2024-05-17T14:42:22.854 INFO:root:teuthology version: 0.0.1.dev276+gcad7edd
2024-05-17T14:42:22.854 DEBUG:teuthology.run:Teuthology command: teuthology --description rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock} --name dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi --verbose --archive /home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217 --owner scheduled_dis@teuthology -- /home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217/orig.config.yaml
2024-05-17T14:42:22.899 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-05-17T14:42:22.996 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217
  branch: wip-dis-testing
  description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack}
    conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/none
    rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}
  email: idryomov@gmail.com
  first_in_suite: false
  job_id: '7711217'
  kernel:
    kdb: true
    sha1: distro
  ktype: distro
  last_in_suite: false
  machine_type: smithi
  meta:
  - desc: run two ceph clusters and install rbd-mirror
  - desc: configure the permissions for client.mirror
  - desc: 2 ceph clusters with 1 mon and 3 osds each
  - desc: run four rbd-mirror daemons per cluster
  - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon
  name: dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 3
      size: 30
  os_type: ubuntu
  os_version: '22.04'
  overrides:
    admin_socket:
      branch: wip-dis-testing
    ceph:
      conf:
        client:
          debug rbd: 20
          debug rbd_mirror: 15
          log to stderr: false
          rbd default features: 125
          rbd mirror image policy type: none
        client.mirror.0:
          admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
          pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
        client.mirror.1:
          admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
          pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
        client.mirror.2:
          admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
          pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
        client.mirror.3:
          admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
          pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
        global:
          mon client directed command retry: 5
          mon warn on pool no app: false
          ms inject socket failures: 5000
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore allocator: stupid
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 1/20
          debug bluestore: 1/20
          debug ms: 1
          debug osd: 20
          debug rocksdb: 4/10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      flavor: default
      fs: xfs
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(OSD_SLOW_PING_TIME
      sha1: ee59d86aad04089481e55c1a15aa5871339935b4
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 1/20
          debug bluestore: 1/20
          debug rocksdb: 4/10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        flavor: default
        sha1: ee59d86aad04089481e55c1a15aa5871339935b4
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
    workunit:
      branch: wip-dis-testing
      sha1: ee59d86aad04089481e55c1a15aa5871339935b4
  owner: scheduled_dis@teuthology
  priority: 70
  repo: https://github.com/ceph/ceph-ci
  roles:
  - - cluster1.mon.a
    - cluster1.mgr.x
    - cluster2.mgr.x
    - cluster1.osd.0
    - cluster1.osd.1
    - cluster1.osd.2
    - cluster1.client.0
    - cluster2.client.0
  - - cluster2.mon.a
    - cluster2.osd.0
    - cluster2.osd.1
    - cluster2.osd.2
    - cluster1.client.mirror
    - cluster1.client.mirror.0
    - cluster1.client.mirror.1
    - cluster1.client.mirror.2
    - cluster1.client.mirror.3
    - cluster1.client.mirror.4
    - cluster1.client.mirror.5
    - cluster1.client.mirror.6
    - cluster2.client.mirror
    - cluster2.client.mirror.0
    - cluster2.client.mirror.1
    - cluster2.client.mirror.2
    - cluster2.client.mirror.3
    - cluster2.client.mirror.4
    - cluster2.client.mirror.5
    - cluster2.client.mirror.6
  seed: 3754
  sha1: ee59d86aad04089481e55c1a15aa5871339935b4
  sleep_before_teardown: 0
  subset: 56/99
  suite: rbd
  suite_branch: wip-dis-testing
  suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa
  suite_relpath: qa
  suite_repo: https://github.com/ceph/ceph-ci
  suite_sha1: ee59d86aad04089481e55c1a15aa5871339935b4
  targets:
    smithi072.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAtFxZM3PX3EqFG5Suxok8lQhwmW0wU9dQiMCx105AEPlaLR+Mmd/eylLsthBupi9TbcCp1qT5241HblZVZk0AQ=
    smithi171.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG6ZXSZlCBgx+DF91uuAf3IuG4mnOVQsHYBVdotBdA7HEH0Zygg0Zvz4SN+EXfOn2SaSabD5rx9Bw8uZG/mFa8k=
  tasks:
  - install:
      extra_packages:
      - rbd-mirror
  - ceph:
      cluster: cluster1
  - ceph:
      cluster: cluster2
  - exec:
      cluster1.client.mirror.0:
      - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer'
        osd 'profile rbd'
      - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer'
        osd 'profile rbd'
      - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror'
        osd 'profile rbd'
      - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror'
        osd 'profile rbd'
  - rbd-mirror:
      client: cluster1.client.mirror.0
      thrash: true
  - rbd-mirror:
      client: cluster1.client.mirror.1
      thrash: true
  - rbd-mirror:
      client: cluster1.client.mirror.2
      thrash: true
  - rbd-mirror:
      client: cluster1.client.mirror.3
      thrash: true
  - rbd-mirror:
      client: cluster2.client.mirror.0
      thrash: true
  - rbd-mirror:
      client: cluster2.client.mirror.1
      thrash: true
  - rbd-mirror:
      client: cluster2.client.mirror.2
      thrash: true
  - rbd-mirror:
      client: cluster2.client.mirror.3
      thrash: true
  - rbd-mirror-thrash:
      cluster: cluster1
  - rbd-mirror-thrash:
      cluster: cluster2
  - workunit:
      clients:
        cluster1.client.mirror:
        - rbd/rbd_mirror_stress.sh
      env:
        CEPH_ARGS: ''
        MIRROR_POOL_MODE: image
        RBD_IMAGE_FEATURES: layering,exclusive-lock
        RBD_MIRROR_INSTANCES: '4'
        RBD_MIRROR_MODE: snapshot
        RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
        RBD_MIRROR_USE_RBD_MIRROR: '1'
      timeout: 6h
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: wip-fix-extra-system-packages
  teuthology_sha1: cad7edd8d899a7c59a48f7da3294fca4cb778f64
  timestamp: 2024-05-17_14:29:27
  tube: smithi
  user: dis
  verbose: true
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885
2024-05-17T14:42:22.996 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa; will attempt to use it
2024-05-17T14:42:22.997 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa/tasks
2024-05-17T14:42:22.997 INFO:teuthology.run_tasks:Running task internal.check_packages...
2024-05-17T14:42:22.999 INFO:teuthology.task.internal:Checking packages...
2024-05-17T14:42:23.017 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'ee59d86aad04089481e55c1a15aa5871339935b4'
2024-05-17T14:42:23.017 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2024-05-17T14:42:23.017 INFO:teuthology.packaging:ref: None
2024-05-17T14:42:23.017 INFO:teuthology.packaging:tag: None
2024-05-17T14:42:23.018 INFO:teuthology.packaging:branch: wip-dis-testing
2024-05-17T14:42:23.018 INFO:teuthology.packaging:sha1: ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:42:23.018 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-dis-testing
2024-05-17T14:42:23.168 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:42:23.169 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2024-05-17T14:42:23.175 INFO:teuthology.task.internal:no buildpackages task found
2024-05-17T14:42:23.176 INFO:teuthology.run_tasks:Running task internal.save_config...
2024-05-17T14:42:23.190 INFO:teuthology.task.internal:Saving configuration
2024-05-17T14:42:23.207 INFO:teuthology.run_tasks:Running task internal.check_lock...
2024-05-17T14:42:23.217 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-05-17T14:42:23.240 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi072.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-17 14:35:28.233298', 'locked_by': 'scheduled_dis@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAMi4BI8DKC5s1EXJN29FH1N+B8Q6QbQzZISxhtKTq+V'}
2024-05-17T14:42:23.257 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi171.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-17 14:35:28.235013', 'locked_by': 'scheduled_dis@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILLPqjLbk+GHflYw605wGyOc5iiWl5g2ime/z5lB9qln'}
2024-05-17T14:42:23.257 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2024-05-17T14:42:23.267 INFO:teuthology.task.internal:roles: ubuntu@smithi072.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0']
2024-05-17T14:42:23.267 INFO:teuthology.task.internal:roles: ubuntu@smithi171.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']
2024-05-17T14:42:23.267 INFO:teuthology.run_tasks:Running task console_log...
2024-05-17T14:42:23.327 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7fc5af6fc400>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7fc5afd87ac0>, signals=[15])
2024-05-17T14:42:23.327 INFO:teuthology.run_tasks:Running task internal.connect...
2024-05-17T14:42:23.336 INFO:teuthology.task.internal:Opening connections...
2024-05-17T14:42:23.336 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi072.front.sepia.ceph.com
2024-05-17T14:42:23.337 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi072.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:42:23.420 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:42:23.421 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:42:23.498 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2024-05-17T14:42:23.507 DEBUG:teuthology.orchestra.run.smithi072:> uname -m
2024-05-17T14:42:23.511 INFO:teuthology.orchestra.run.smithi072.stdout:x86_64
2024-05-17T14:42:23.512 DEBUG:teuthology.orchestra.run.smithi072:> cat /etc/os-release
2024-05-17T14:42:23.557 INFO:teuthology.orchestra.run.smithi072.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:NAME="Ubuntu"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:VERSION_ID="22.04"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:VERSION_CODENAME=jammy
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:ID=ubuntu
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:ID_LIKE=debian
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:HOME_URL="https://www.ubuntu.com/"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:SUPPORT_URL="https://help.ubuntu.com/"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
2024-05-17T14:42:23.558 INFO:teuthology.orchestra.run.smithi072.stdout:UBUNTU_CODENAME=jammy
2024-05-17T14:42:23.559 INFO:teuthology.lock.ops:Updating smithi072.front.sepia.ceph.com on lock server
2024-05-17T14:42:23.586 DEBUG:teuthology.orchestra.run.smithi171:> uname -m
2024-05-17T14:42:23.591 INFO:teuthology.orchestra.run.smithi171.stdout:x86_64
2024-05-17T14:42:23.591 DEBUG:teuthology.orchestra.run.smithi171:> cat /etc/os-release
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS"
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:NAME="Ubuntu"
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION_ID="22.04"
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)"
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION_CODENAME=jammy
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:ID=ubuntu
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:ID_LIKE=debian
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:HOME_URL="https://www.ubuntu.com/"
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:SUPPORT_URL="https://help.ubuntu.com/"
2024-05-17T14:42:23.643 INFO:teuthology.orchestra.run.smithi171.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
2024-05-17T14:42:23.644 INFO:teuthology.orchestra.run.smithi171.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
2024-05-17T14:42:23.644 INFO:teuthology.orchestra.run.smithi171.stdout:UBUNTU_CODENAME=jammy
2024-05-17T14:42:23.644 INFO:teuthology.lock.ops:Updating smithi171.front.sepia.ceph.com on lock server
2024-05-17T14:42:23.669 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2024-05-17T14:42:23.681 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2024-05-17T14:42:23.692 INFO:teuthology.task.internal:Checking for old test directory...
2024-05-17T14:42:23.692 DEBUG:teuthology.orchestra.run.smithi072:> test '!' -e /home/ubuntu/cephtest
2024-05-17T14:42:23.694 DEBUG:teuthology.orchestra.run.smithi171:> test '!' -e /home/ubuntu/cephtest
2024-05-17T14:42:23.699 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2024-05-17T14:42:23.708 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2024-05-17T14:42:23.708 DEBUG:teuthology.orchestra.run.smithi072:> test -z $(ls -A /var/lib/ceph)
2024-05-17T14:42:23.740 DEBUG:teuthology.orchestra.run.smithi171:> test -z $(ls -A /var/lib/ceph)
2024-05-17T14:42:23.749 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2024-05-17T14:42:23.807 INFO:teuthology.run_tasks:Running task kernel...
2024-05-17T14:42:23.824 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'}
2024-05-17T14:42:23.824 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'kdb': True, 'sha1': 'distro'}}, timeout 300
2024-05-17T14:42:23.824 DEBUG:teuthology.orchestra.run.smithi072:> test -f /run/.containerenv -o -f /.dockerenv
2024-05-17T14:42:23.825 DEBUG:teuthology.orchestra.run.smithi171:> test -f /run/.containerenv -o -f /.dockerenv
2024-05-17T14:42:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:42:23.829 DEBUG:teuthology.orchestra.run.smithi072:> uname -r
2024-05-17T14:42:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:42:23.830 DEBUG:teuthology.orchestra.run.smithi171:> uname -r
2024-05-17T14:42:23.875 INFO:teuthology.orchestra.run.smithi072.stdout:5.15.0-56-generic
2024-05-17T14:42:23.875 INFO:teuthology.task.kernel:Running kernel on smithi072: 5.15.0-56-generic
2024-05-17T14:42:23.875 DEBUG:teuthology.orchestra.run.smithi072:> sudo apt-get clean
2024-05-17T14:42:23.880 INFO:teuthology.orchestra.run.smithi171.stdout:5.15.0-56-generic
2024-05-17T14:42:23.880 INFO:teuthology.task.kernel:Running kernel on smithi171: 5.15.0-56-generic
2024-05-17T14:42:23.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo apt-get clean
2024-05-17T14:42:24.013 DEBUG:teuthology.orchestra.run.smithi072:> sudo apt-get update
2024-05-17T14:42:24.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo apt-get update
2024-05-17T14:42:24.207 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease
2024-05-17T14:42:24.216 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease
2024-05-17T14:42:24.216 INFO:teuthology.orchestra.run.smithi072.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]
2024-05-17T14:42:24.238 INFO:teuthology.orchestra.run.smithi072.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]
2024-05-17T14:42:24.253 INFO:teuthology.orchestra.run.smithi171.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]
2024-05-17T14:42:24.273 INFO:teuthology.orchestra.run.smithi171.stdout:Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]
2024-05-17T14:42:24.452 INFO:teuthology.orchestra.run.smithi072.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB]
2024-05-17T14:42:24.462 INFO:teuthology.orchestra.run.smithi072.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,467 kB]
2024-05-17T14:42:24.493 INFO:teuthology.orchestra.run.smithi171.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB]
2024-05-17T14:42:24.572 INFO:teuthology.orchestra.run.smithi072.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [631 kB]
2024-05-17T14:42:24.612 INFO:teuthology.orchestra.run.smithi171.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,678 kB]
2024-05-17T14:42:24.701 INFO:teuthology.orchestra.run.smithi171.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [470 kB]
2024-05-17T14:42:24.805 INFO:teuthology.orchestra.run.smithi072.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,678 kB]
2024-05-17T14:42:24.860 INFO:teuthology.orchestra.run.smithi072.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [470 kB]
2024-05-17T14:42:24.884 INFO:teuthology.orchestra.run.smithi072.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [252 kB]
2024-05-17T14:42:24.944 INFO:teuthology.orchestra.run.smithi072.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [311 kB]
2024-05-17T14:42:24.945 INFO:teuthology.orchestra.run.smithi072.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB]
2024-05-17T14:42:24.945 INFO:teuthology.orchestra.run.smithi072.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,866 kB]
2024-05-17T14:42:24.949 INFO:teuthology.orchestra.run.smithi171.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [631 kB]
2024-05-17T14:42:24.955 INFO:teuthology.orchestra.run.smithi072.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB]
2024-05-17T14:42:24.955 INFO:teuthology.orchestra.run.smithi072.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,923 kB]
2024-05-17T14:42:24.973 INFO:teuthology.orchestra.run.smithi171.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [311 kB]
2024-05-17T14:42:24.976 INFO:teuthology.orchestra.run.smithi072.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [36.5 kB]
2024-05-17T14:42:24.976 INFO:teuthology.orchestra.run.smithi072.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [317 kB]
2024-05-17T14:42:24.978 INFO:teuthology.orchestra.run.smithi171.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB]
2024-05-17T14:42:24.979 INFO:teuthology.orchestra.run.smithi171.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,923 kB]
2024-05-17T14:42:24.983 INFO:teuthology.orchestra.run.smithi072.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B]
2024-05-17T14:42:24.984 INFO:teuthology.orchestra.run.smithi072.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [603 kB]
2024-05-17T14:42:24.996 INFO:teuthology.orchestra.run.smithi072.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [38.1 kB]
2024-05-17T14:42:24.997 INFO:teuthology.orchestra.run.smithi171.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,467 kB]
2024-05-17T14:42:25.023 INFO:teuthology.orchestra.run.smithi072.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [327 kB]
2024-05-17T14:42:25.023 INFO:teuthology.orchestra.run.smithi072.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [852 kB]
2024-05-17T14:42:25.030 INFO:teuthology.orchestra.run.smithi072.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B]
2024-05-17T14:42:25.030 INFO:teuthology.orchestra.run.smithi072.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [702 kB]
2024-05-17T14:42:25.033 INFO:teuthology.orchestra.run.smithi072.stdout:Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [164 kB]
2024-05-17T14:42:25.035 INFO:teuthology.orchestra.run.smithi072.stdout:Get:25 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB]
2024-05-17T14:42:25.035 INFO:teuthology.orchestra.run.smithi072.stdout:Get:26 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B]
2024-05-17T14:42:25.035 INFO:teuthology.orchestra.run.smithi072.stdout:Get:27 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB]
2024-05-17T14:42:25.036 INFO:teuthology.orchestra.run.smithi171.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [38.1 kB]
2024-05-17T14:42:25.036 INFO:teuthology.orchestra.run.smithi171.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [327 kB]
2024-05-17T14:42:25.042 INFO:teuthology.orchestra.run.smithi072.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,074 kB]
2024-05-17T14:42:25.044 INFO:teuthology.orchestra.run.smithi171.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B]
2024-05-17T14:42:25.044 INFO:teuthology.orchestra.run.smithi171.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [702 kB]
2024-05-17T14:42:25.044 INFO:teuthology.orchestra.run.smithi072.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B]
2024-05-17T14:42:25.044 INFO:teuthology.orchestra.run.smithi072.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B]
2024-05-17T14:42:25.061 INFO:teuthology.orchestra.run.smithi171.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,074 kB]
2024-05-17T14:42:25.078 INFO:teuthology.orchestra.run.smithi171.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [246 kB]
2024-05-17T14:42:25.101 INFO:teuthology.orchestra.run.smithi171.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB]
2024-05-17T14:42:25.102 INFO:teuthology.orchestra.run.smithi171.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [42.7 kB]
2024-05-17T14:42:25.103 INFO:teuthology.orchestra.run.smithi171.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,472 B]
2024-05-17T14:42:25.103 INFO:teuthology.orchestra.run.smithi171.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.4 kB]
2024-05-17T14:42:25.103 INFO:teuthology.orchestra.run.smithi171.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B]
2024-05-17T14:42:25.104 INFO:teuthology.orchestra.run.smithi171.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.3 kB]
2024-05-17T14:42:25.105 INFO:teuthology.orchestra.run.smithi171.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB]
2024-05-17T14:42:25.107 INFO:teuthology.orchestra.run.smithi171.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB]
2024-05-17T14:42:25.107 INFO:teuthology.orchestra.run.smithi171.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B]
2024-05-17T14:42:25.127 INFO:teuthology.orchestra.run.smithi171.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [16.0 kB]
2024-05-17T14:42:25.128 INFO:teuthology.orchestra.run.smithi171.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [27.2 kB]
2024-05-17T14:42:25.128 INFO:teuthology.orchestra.run.smithi171.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.2 kB]
2024-05-17T14:42:25.128 INFO:teuthology.orchestra.run.smithi171.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B]
2024-05-17T14:42:25.166 INFO:teuthology.orchestra.run.smithi171.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [252 kB]
2024-05-17T14:42:25.179 INFO:teuthology.orchestra.run.smithi171.stdout:Get:32 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB]
2024-05-17T14:42:25.179 INFO:teuthology.orchestra.run.smithi171.stdout:Get:33 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,866 kB]
2024-05-17T14:42:25.238 INFO:teuthology.orchestra.run.smithi072.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [246 kB]
2024-05-17T14:42:25.262 INFO:teuthology.orchestra.run.smithi171.stdout:Get:34 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [36.5 kB]
2024-05-17T14:42:25.263 INFO:teuthology.orchestra.run.smithi171.stdout:Get:35 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [317 kB]
2024-05-17T14:42:25.271 INFO:teuthology.orchestra.run.smithi171.stdout:Get:36 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B]
2024-05-17T14:42:25.272 INFO:teuthology.orchestra.run.smithi171.stdout:Get:37 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [852 kB]
2024-05-17T14:42:25.292 INFO:teuthology.orchestra.run.smithi171.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [603 kB]
2024-05-17T14:42:25.470 INFO:teuthology.orchestra.run.smithi171.stdout:Get:39 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [164 kB]
2024-05-17T14:42:25.684 INFO:teuthology.orchestra.run.smithi072.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB]
2024-05-17T14:42:25.786 INFO:teuthology.orchestra.run.smithi072.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,472 B]
2024-05-17T14:42:25.866 INFO:teuthology.orchestra.run.smithi171.stdout:Get:40 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB]
2024-05-17T14:42:25.868 INFO:teuthology.orchestra.run.smithi072.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [42.7 kB]
2024-05-17T14:42:25.872 INFO:teuthology.orchestra.run.smithi171.stdout:Get:41 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB]
2024-05-17T14:42:25.885 INFO:teuthology.orchestra.run.smithi171.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B]
2024-05-17T14:42:25.886 INFO:teuthology.orchestra.run.smithi171.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B]
2024-05-17T14:42:25.889 INFO:teuthology.orchestra.run.smithi171.stdout:Get:44 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B]
2024-05-17T14:42:25.960 INFO:teuthology.orchestra.run.smithi072.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.4 kB]
2024-05-17T14:42:26.044 INFO:teuthology.orchestra.run.smithi072.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B]
2024-05-17T14:42:26.125 INFO:teuthology.orchestra.run.smithi072.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.3 kB]
2024-05-17T14:42:26.222 INFO:teuthology.orchestra.run.smithi072.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB]
2024-05-17T14:42:26.320 INFO:teuthology.orchestra.run.smithi072.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB]
2024-05-17T14:42:26.404 INFO:teuthology.orchestra.run.smithi072.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B]
2024-05-17T14:42:26.486 INFO:teuthology.orchestra.run.smithi072.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [27.2 kB]
2024-05-17T14:42:26.656 INFO:teuthology.orchestra.run.smithi072.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [16.0 kB]
2024-05-17T14:42:26.819 INFO:teuthology.orchestra.run.smithi072.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.2 kB]
2024-05-17T14:42:26.929 INFO:teuthology.orchestra.run.smithi072.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B]
2024-05-17T14:42:30.883 INFO:teuthology.orchestra.run.smithi171.stdout:Fetched 13.7 MB in 2s (5,900 kB/s)
2024-05-17T14:42:31.315 INFO:teuthology.orchestra.run.smithi072.stdout:Fetched 13.7 MB in 3s (4,800 kB/s)
2024-05-17T14:42:32.307 INFO:teuthology.orchestra.run.smithi171.stdout:Reading package lists...
2024-05-17T14:42:32.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic
2024-05-17T14:42:32.384 INFO:teuthology.orchestra.run.smithi171.stdout:Reading package lists...
2024-05-17T14:42:32.584 INFO:teuthology.orchestra.run.smithi171.stdout:Building dependency tree...
2024-05-17T14:42:32.585 INFO:teuthology.orchestra.run.smithi171.stdout:Reading state information...
2024-05-17T14:42:32.760 INFO:teuthology.orchestra.run.smithi171.stdout:The following packages were automatically installed and are no longer required:
2024-05-17T14:42:32.760 INFO:teuthology.orchestra.run.smithi171.stdout:  libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0
2024-05-17T14:42:32.761 INFO:teuthology.orchestra.run.smithi171.stdout:  libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib
2024-05-17T14:42:32.761 INFO:teuthology.orchestra.run.smithi171.stdout:  libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib
2024-05-17T14:42:32.761 INFO:teuthology.orchestra.run.smithi171.stdout:  python2 python2-dev python2-minimal python2.7 python2.7-dev
2024-05-17T14:42:32.761 INFO:teuthology.orchestra.run.smithi171.stdout:  python2.7-minimal
2024-05-17T14:42:32.761 INFO:teuthology.orchestra.run.smithi171.stdout:Use 'sudo apt autoremove' to remove them.
2024-05-17T14:42:32.762 INFO:teuthology.orchestra.run.smithi171.stdout:The following additional packages will be installed:
2024-05-17T14:42:32.762 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-generic linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic
2024-05-17T14:42:32.762 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-headers-generic linux-image-5.15.0-107-generic
2024-05-17T14:42:32.763 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-modules-5.15.0-107-generic linux-modules-extra-5.15.0-107-generic
2024-05-17T14:42:32.764 INFO:teuthology.orchestra.run.smithi171.stdout:Suggested packages:
2024-05-17T14:42:32.764 INFO:teuthology.orchestra.run.smithi171.stdout:  fdutils linux-doc | linux-source-5.15.0 linux-tools
2024-05-17T14:42:32.764 INFO:teuthology.orchestra.run.smithi171.stdout:Recommended packages:
2024-05-17T14:42:32.764 INFO:teuthology.orchestra.run.smithi171.stdout:  thermald
2024-05-17T14:42:32.802 INFO:teuthology.orchestra.run.smithi171.stdout:The following NEW packages will be installed:
2024-05-17T14:42:32.803 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic
2024-05-17T14:42:32.803 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-image-5.15.0-107-generic linux-modules-5.15.0-107-generic
2024-05-17T14:42:32.803 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-modules-extra-5.15.0-107-generic
2024-05-17T14:42:32.804 INFO:teuthology.orchestra.run.smithi171.stdout:The following packages will be upgraded:
2024-05-17T14:42:32.805 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-generic linux-headers-generic linux-image-generic
2024-05-17T14:42:32.864 INFO:teuthology.orchestra.run.smithi171.stdout:3 upgraded, 5 newly installed, 0 to remove and 320 not upgraded.
2024-05-17T14:42:32.864 INFO:teuthology.orchestra.run.smithi171.stdout:Need to get 113 MB of archives.
2024-05-17T14:42:32.864 INFO:teuthology.orchestra.run.smithi171.stdout:After this operation, 583 MB of additional disk space will be used.
2024-05-17T14:42:32.864 INFO:teuthology.orchestra.run.smithi171.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-107-generic amd64 5.15.0-107.117 [22.7 MB]
2024-05-17T14:42:33.502 INFO:teuthology.orchestra.run.smithi171.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-107-generic amd64 5.15.0-107.117 [11.5 MB]
2024-05-17T14:42:33.646 INFO:teuthology.orchestra.run.smithi072.stdout:Reading package lists...
2024-05-17T14:42:33.665 DEBUG:teuthology.orchestra.run.smithi072:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic
2024-05-17T14:42:33.701 INFO:teuthology.orchestra.run.smithi171.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-107-generic amd64 5.15.0-107.117 [63.9 MB]
2024-05-17T14:42:33.723 INFO:teuthology.orchestra.run.smithi072.stdout:Reading package lists...
2024-05-17T14:42:33.927 INFO:teuthology.orchestra.run.smithi072.stdout:Building dependency tree...
2024-05-17T14:42:33.927 INFO:teuthology.orchestra.run.smithi072.stdout:Reading state information...
2024-05-17T14:42:34.091 INFO:teuthology.orchestra.run.smithi072.stdout:The following packages were automatically installed and are no longer required:
2024-05-17T14:42:34.091 INFO:teuthology.orchestra.run.smithi072.stdout:  libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0
2024-05-17T14:42:34.091 INFO:teuthology.orchestra.run.smithi072.stdout:  libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib
2024-05-17T14:42:34.091 INFO:teuthology.orchestra.run.smithi072.stdout:  libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib
2024-05-17T14:42:34.092 INFO:teuthology.orchestra.run.smithi072.stdout:  python2 python2-dev python2-minimal python2.7 python2.7-dev
2024-05-17T14:42:34.092 INFO:teuthology.orchestra.run.smithi072.stdout:  python2.7-minimal
2024-05-17T14:42:34.092 INFO:teuthology.orchestra.run.smithi072.stdout:Use 'sudo apt autoremove' to remove them.
2024-05-17T14:42:34.092 INFO:teuthology.orchestra.run.smithi072.stdout:The following additional packages will be installed:
2024-05-17T14:42:34.093 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-generic linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic
2024-05-17T14:42:34.093 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-headers-generic linux-image-5.15.0-107-generic
2024-05-17T14:42:34.093 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-modules-5.15.0-107-generic linux-modules-extra-5.15.0-107-generic
2024-05-17T14:42:34.094 INFO:teuthology.orchestra.run.smithi072.stdout:Suggested packages:
2024-05-17T14:42:34.094 INFO:teuthology.orchestra.run.smithi072.stdout:  fdutils linux-doc | linux-source-5.15.0 linux-tools
2024-05-17T14:42:34.094 INFO:teuthology.orchestra.run.smithi072.stdout:Recommended packages:
2024-05-17T14:42:34.094 INFO:teuthology.orchestra.run.smithi072.stdout:  thermald
2024-05-17T14:42:34.147 INFO:teuthology.orchestra.run.smithi072.stdout:The following NEW packages will be installed:
2024-05-17T14:42:34.148 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic
2024-05-17T14:42:34.148 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-image-5.15.0-107-generic linux-modules-5.15.0-107-generic
2024-05-17T14:42:34.148 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-modules-extra-5.15.0-107-generic
2024-05-17T14:42:34.149 INFO:teuthology.orchestra.run.smithi072.stdout:The following packages will be upgraded:
2024-05-17T14:42:34.150 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-generic linux-headers-generic linux-image-generic
2024-05-17T14:42:34.262 INFO:teuthology.orchestra.run.smithi072.stdout:3 upgraded, 5 newly installed, 0 to remove and 320 not upgraded.
2024-05-17T14:42:34.262 INFO:teuthology.orchestra.run.smithi072.stdout:Need to get 113 MB of archives.
2024-05-17T14:42:34.262 INFO:teuthology.orchestra.run.smithi072.stdout:After this operation, 583 MB of additional disk space will be used.
2024-05-17T14:42:34.262 INFO:teuthology.orchestra.run.smithi072.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-107-generic amd64 5.15.0-107.117 [22.7 MB]
2024-05-17T14:42:35.167 INFO:teuthology.orchestra.run.smithi171.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.107.107 [1,694 B]
2024-05-17T14:42:35.168 INFO:teuthology.orchestra.run.smithi171.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.107.107 [2,496 B]
2024-05-17T14:42:35.168 INFO:teuthology.orchestra.run.smithi171.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-107 all 5.15.0-107.117 [12.3 MB]
2024-05-17T14:42:35.204 INFO:teuthology.orchestra.run.smithi072.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-107-generic amd64 5.15.0-107.117 [11.5 MB]
2024-05-17T14:42:35.466 INFO:teuthology.orchestra.run.smithi072.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-107-generic amd64 5.15.0-107.117 [63.9 MB]
2024-05-17T14:42:35.501 INFO:teuthology.orchestra.run.smithi171.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-107-generic amd64 5.15.0-107.117 [2,887 kB]
2024-05-17T14:42:35.637 INFO:teuthology.orchestra.run.smithi171.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.107.107 [2,354 B]
2024-05-17T14:42:36.032 INFO:teuthology.orchestra.run.smithi171.stdout:Fetched 113 MB in 3s (40.1 MB/s)
2024-05-17T14:42:36.207 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package linux-modules-5.15.0-107-generic.
2024-05-17T14:42:37.172 INFO:teuthology.orchestra.run.smithi072.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.107.107 [1,694 B]
2024-05-17T14:42:37.172 INFO:teuthology.orchestra.run.smithi072.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.107.107 [2,496 B]
2024-05-17T14:42:37.173 INFO:teuthology.orchestra.run.smithi072.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-107 all 5.15.0-107.117 [12.3 MB]
2024-05-17T14:42:37.539 INFO:teuthology.orchestra.run.smithi072.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-107-generic amd64 5.15.0-107.117 [2,887 kB]
2024-05-17T14:42:37.620 INFO:teuthology.orchestra.run.smithi072.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.107.107 [2,354 B]
2024-05-17T14:42:38.016 INFO:teuthology.orchestra.run.smithi072.stdout:Fetched 113 MB in 3s (32.7 MB/s)
2024-05-17T14:42:38.237 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package linux-modules-5.15.0-107-generic.
2024-05-17T14:42:38.241 INFO:teuthology.orchestra.run.smithi171.stdout:(Reading database ... 
(Reading database ... 5%
(Reading database ... 10%
(Reading database ... 15%
(Reading database ... 20%
(Reading database ... 25%
(Reading database ... 30%
(Reading database ... 35%
(Reading database ... 40%
(Reading database ... 45%
(Reading database ... 50%
(Reading database ... 55%
(Reading database ... 60%
(Reading database ... 65%
(Reading database ... 70%
(Reading database ... 75%
(Reading database ... 80%
(Reading database ... 85%
(Reading database ... 90%
(Reading database ... 95%
(Reading database ... 100%
(Reading database ... 129542 files and directories currently installed.)
2024-05-17T14:42:38.246 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../0-linux-modules-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:38.324 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-modules-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:40.541 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package linux-image-5.15.0-107-generic.
2024-05-17T14:42:40.562 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../1-linux-image-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:40.651 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-image-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:41.179 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-107-generic.
2024-05-17T14:42:41.200 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:41.336 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-modules-extra-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:41.370 INFO:teuthology.orchestra.run.smithi072.stdout:(Reading database ... 
(Reading database ... 5%
(Reading database ... 10%
(Reading database ... 15%
(Reading database ... 20%
(Reading database ... 25%
(Reading database ... 30%
(Reading database ... 35%
(Reading database ... 40%
(Reading database ... 45%
(Reading database ... 50%
(Reading database ... 55%
(Reading database ... 60%
(Reading database ... 65%
(Reading database ... 70%
(Reading database ... 75%
(Reading database ... 80%
(Reading database ... 85%
(Reading database ... 90%
(Reading database ... 95%
(Reading database ... 100%
(Reading database ... 129542 files and directories currently installed.)
2024-05-17T14:42:41.375 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../0-linux-modules-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:41.453 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-modules-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:44.700 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package linux-image-5.15.0-107-generic.
2024-05-17T14:42:44.721 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../1-linux-image-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:44.824 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-image-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:45.366 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-107-generic.
2024-05-17T14:42:45.387 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:45.424 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-modules-extra-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:47.711 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../3-linux-generic_5.15.0.107.107_amd64.deb ...
2024-05-17T14:42:47.837 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-generic (5.15.0.107.107) over (5.15.0.56.54) ...
2024-05-17T14:42:48.161 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.107.107_amd64.deb ...
2024-05-17T14:42:48.258 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-image-generic (5.15.0.107.107) over (5.15.0.56.54) ...
2024-05-17T14:42:48.578 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package linux-headers-5.15.0-107.
2024-05-17T14:42:48.588 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../5-linux-headers-5.15.0-107_5.15.0-107.117_all.deb ...
2024-05-17T14:42:48.628 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-headers-5.15.0-107 (5.15.0-107.117) ...
2024-05-17T14:42:53.314 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../3-linux-generic_5.15.0.107.107_amd64.deb ...
2024-05-17T14:42:53.434 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-generic (5.15.0.107.107) over (5.15.0.56.54) ...
2024-05-17T14:42:53.769 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.107.107_amd64.deb ...
2024-05-17T14:42:53.901 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-image-generic (5.15.0.107.107) over (5.15.0.56.54) ...
2024-05-17T14:42:54.222 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package linux-headers-5.15.0-107.
2024-05-17T14:42:54.244 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../5-linux-headers-5.15.0-107_5.15.0-107.117_all.deb ...
2024-05-17T14:42:54.375 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-headers-5.15.0-107 (5.15.0-107.117) ...
2024-05-17T14:42:54.476 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package linux-headers-5.15.0-107-generic.
2024-05-17T14:42:54.513 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../6-linux-headers-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:42:54.550 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-headers-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:57.236 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.107.107_amd64.deb ...
2024-05-17T14:42:57.444 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking linux-headers-generic (5.15.0.107.107) over (5.15.0.56.54) ...
2024-05-17T14:42:57.769 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-headers-5.15.0-107 (5.15.0-107.117) ...
2024-05-17T14:42:57.908 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-headers-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:58.009 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-headers-generic (5.15.0.107.107) ...
2024-05-17T14:42:58.138 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-image-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:42:59.430 INFO:teuthology.orchestra.run.smithi171.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic
2024-05-17T14:42:59.431 INFO:teuthology.orchestra.run.smithi171.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic
2024-05-17T14:42:59.431 INFO:teuthology.orchestra.run.smithi171.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-107-generic
2024-05-17T14:42:59.431 INFO:teuthology.orchestra.run.smithi171.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-107-generic
2024-05-17T14:42:59.582 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-modules-extra-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:00.999 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-modules-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:02.264 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package linux-headers-5.15.0-107-generic.
2024-05-17T14:43:02.301 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../6-linux-headers-5.15.0-107-generic_5.15.0-107.117_amd64.deb ...
2024-05-17T14:43:02.339 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-headers-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:03.734 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-image-generic (5.15.0.107.107) ...
2024-05-17T14:43:03.976 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up linux-generic (5.15.0.107.107) ...
2024-05-17T14:43:04.103 INFO:teuthology.orchestra.run.smithi171.stdout:Processing triggers for linux-image-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:04.160 INFO:teuthology.orchestra.run.smithi171.stdout:/etc/kernel/postinst.d/initramfs-tools:
2024-05-17T14:43:04.161 INFO:teuthology.orchestra.run.smithi171.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:04.726 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.107.107_amd64.deb ...
2024-05-17T14:43:04.844 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking linux-headers-generic (5.15.0.107.107) over (5.15.0.56.54) ...
2024-05-17T14:43:05.248 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-headers-5.15.0-107 (5.15.0-107.117) ...
2024-05-17T14:43:05.379 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-headers-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:05.514 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-headers-generic (5.15.0.107.107) ...
2024-05-17T14:43:05.655 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-image-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:07.422 INFO:teuthology.orchestra.run.smithi072.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic
2024-05-17T14:43:07.422 INFO:teuthology.orchestra.run.smithi072.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic
2024-05-17T14:43:07.422 INFO:teuthology.orchestra.run.smithi072.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-107-generic
2024-05-17T14:43:07.423 INFO:teuthology.orchestra.run.smithi072.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:08.221 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-modules-extra-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:09.608 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-modules-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:11.636 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-image-generic (5.15.0.107.107) ...
2024-05-17T14:43:11.762 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up linux-generic (5.15.0.107.107) ...
2024-05-17T14:43:11.899 INFO:teuthology.orchestra.run.smithi072.stdout:Processing triggers for linux-image-5.15.0-107-generic (5.15.0-107.117) ...
2024-05-17T14:43:11.969 INFO:teuthology.orchestra.run.smithi072.stdout:/etc/kernel/postinst.d/initramfs-tools:
2024-05-17T14:43:11.969 INFO:teuthology.orchestra.run.smithi072.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:18.364 INFO:teuthology.orchestra.run.smithi171.stdout:/etc/kernel/postinst.d/zz-update-grub:
2024-05-17T14:43:18.364 INFO:teuthology.orchestra.run.smithi171.stdout:Sourcing file `/etc/default/grub'
2024-05-17T14:43:18.378 INFO:teuthology.orchestra.run.smithi171.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg'
2024-05-17T14:43:18.398 INFO:teuthology.orchestra.run.smithi171.stdout:Generating grub configuration file ...
2024-05-17T14:43:18.661 INFO:teuthology.orchestra.run.smithi171.stdout:Found linux image: /boot/vmlinuz-5.15.0-107-generic
2024-05-17T14:43:18.673 INFO:teuthology.orchestra.run.smithi171.stdout:Found initrd image: /boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:18.965 INFO:teuthology.orchestra.run.smithi171.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic
2024-05-17T14:43:18.970 INFO:teuthology.orchestra.run.smithi171.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic
2024-05-17T14:43:19.049 INFO:teuthology.orchestra.run.smithi171.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic
2024-05-17T14:43:19.054 INFO:teuthology.orchestra.run.smithi171.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic
2024-05-17T14:43:19.213 INFO:teuthology.orchestra.run.smithi171.stdout:Warning: os-prober will not be executed to detect other bootable partitions.
2024-05-17T14:43:19.213 INFO:teuthology.orchestra.run.smithi171.stdout:Systems on them will not be added to the GRUB boot configuration.
2024-05-17T14:43:19.214 INFO:teuthology.orchestra.run.smithi171.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry.
2024-05-17T14:43:19.248 INFO:teuthology.orchestra.run.smithi171.stdout:done
2024-05-17T14:43:19.706 DEBUG:teuthology.orchestra.run.smithi171:> dpkg -s linux-image-generic
2024-05-17T14:43:19.734 INFO:teuthology.orchestra.run.smithi171.stdout:Package: linux-image-generic
2024-05-17T14:43:19.734 INFO:teuthology.orchestra.run.smithi171.stdout:Status: install ok installed
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Priority: optional
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Section: kernel
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Installed-Size: 21
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Maintainer: Ubuntu Kernel Team <kernel-team@lists.ubuntu.com>
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Architecture: amd64
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Source: linux-meta
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Version: 5.15.0.107.107
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-107), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3)
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Depends: linux-image-5.15.0-107-generic, linux-modules-extra-5.15.0-107-generic, linux-firmware, intel-microcode, amd64-microcode
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Recommends: thermald
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout:Description: Generic Linux kernel image
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout: This package will always depend on the latest generic kernel image
2024-05-17T14:43:19.735 INFO:teuthology.orchestra.run.smithi171.stdout: available.
2024-05-17T14:43:19.736 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-107-generic
2024-05-17T14:43:19.736 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2024-05-17T14:43:19.736 DEBUG:teuthology.task.kernel:src is distro, skipping download
2024-05-17T14:43:19.736 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a...
2024-05-17T14:43:19.737 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi171.front.sepia.ceph.com, path=None, version=distro)
2024-05-17T14:43:19.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo apt-get clean
2024-05-17T14:43:19.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo apt-get update
2024-05-17T14:43:20.182 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease
2024-05-17T14:43:20.191 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease
2024-05-17T14:43:20.249 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease
2024-05-17T14:43:20.331 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease
2024-05-17T14:43:21.607 INFO:teuthology.orchestra.run.smithi171.stdout:Reading package lists...
2024-05-17T14:43:21.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic
2024-05-17T14:43:21.688 INFO:teuthology.orchestra.run.smithi171.stdout:Reading package lists...
2024-05-17T14:43:21.898 INFO:teuthology.orchestra.run.smithi171.stdout:Building dependency tree...
2024-05-17T14:43:21.898 INFO:teuthology.orchestra.run.smithi171.stdout:Reading state information...
2024-05-17T14:43:22.059 INFO:teuthology.orchestra.run.smithi171.stdout:linux-image-generic is already the newest version (5.15.0.107.107).
2024-05-17T14:43:22.059 INFO:teuthology.orchestra.run.smithi171.stdout:The following packages were automatically installed and are no longer required:
2024-05-17T14:43:22.059 INFO:teuthology.orchestra.run.smithi171.stdout:  libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0
2024-05-17T14:43:22.059 INFO:teuthology.orchestra.run.smithi171.stdout:  libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib
2024-05-17T14:43:22.059 INFO:teuthology.orchestra.run.smithi171.stdout:  libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib
2024-05-17T14:43:22.060 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2
2024-05-17T14:43:22.060 INFO:teuthology.orchestra.run.smithi171.stdout:  python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal
2024-05-17T14:43:22.060 INFO:teuthology.orchestra.run.smithi171.stdout:Use 'sudo apt autoremove' to remove them.
2024-05-17T14:43:22.098 INFO:teuthology.orchestra.run.smithi171.stdout:0 upgraded, 0 newly installed, 0 to remove and 320 not upgraded.
2024-05-17T14:43:22.100 DEBUG:teuthology.orchestra.run.smithi171:> dpkg -s linux-image-generic
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Package: linux-image-generic
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Status: install ok installed
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Priority: optional
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Section: kernel
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Installed-Size: 21
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Maintainer: Ubuntu Kernel Team <kernel-team@lists.ubuntu.com>
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Architecture: amd64
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Source: linux-meta
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Version: 5.15.0.107.107
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-107), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3)
2024-05-17T14:43:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:Depends: linux-image-5.15.0-107-generic, linux-modules-extra-5.15.0-107-generic, linux-firmware, intel-microcode, amd64-microcode
2024-05-17T14:43:22.113 INFO:teuthology.orchestra.run.smithi171.stdout:Recommends: thermald
2024-05-17T14:43:22.113 INFO:teuthology.orchestra.run.smithi171.stdout:Description: Generic Linux kernel image
2024-05-17T14:43:22.113 INFO:teuthology.orchestra.run.smithi171.stdout: This package will always depend on the latest generic kernel image
2024-05-17T14:43:22.113 INFO:teuthology.orchestra.run.smithi171.stdout: available.
2024-05-17T14:43:22.113 DEBUG:teuthology.orchestra.run.smithi171:> mktemp
2024-05-17T14:43:22.157 INFO:teuthology.orchestra.run.smithi171.stdout:/tmp/tmp.3PDGvOyUFd
2024-05-17T14:43:22.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo cp /boot/grub/grub.cfg /tmp/tmp.3PDGvOyUFd
2024-05-17T14:43:22.210 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 0666 /tmp/tmp.3PDGvOyUFd
2024-05-17T14:43:22.324 DEBUG:teuthology.orchestra.remote:smithi171:/tmp/tmp.3PDGvOyUFd is 10KB
2024-05-17T14:43:22.336 DEBUG:teuthology.orchestra.run.smithi171:> rm -fr /tmp/tmp.3PDGvOyUFd
2024-05-17T14:43:22.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /etc/grub.d/01_ceph_kernel
2024-05-17T14:43:22.391 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:43:22.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/grub.d/01_ceph_kernel
2024-05-17T14:43:22.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 755 /etc/grub.d/01_ceph_kernel
2024-05-17T14:43:22.487 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-107-generic
2024-05-17T14:43:22.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo update-grub
2024-05-17T14:43:23.714 INFO:teuthology.orchestra.run.smithi171.stderr:Sourcing file `/etc/default/grub'
2024-05-17T14:43:23.715 INFO:teuthology.orchestra.run.smithi171.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg'
2024-05-17T14:43:23.717 INFO:teuthology.orchestra.run.smithi171.stderr:Generating grub configuration file ...
2024-05-17T14:43:23.893 INFO:teuthology.orchestra.run.smithi171.stderr:Found linux image: /boot/vmlinuz-5.15.0-107-generic
2024-05-17T14:43:23.899 INFO:teuthology.orchestra.run.smithi171.stderr:Found initrd image: /boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:24.149 INFO:teuthology.orchestra.run.smithi171.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic
2024-05-17T14:43:24.154 INFO:teuthology.orchestra.run.smithi171.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic
2024-05-17T14:43:24.195 INFO:teuthology.orchestra.run.smithi171.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic
2024-05-17T14:43:24.201 INFO:teuthology.orchestra.run.smithi171.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic
2024-05-17T14:43:24.320 INFO:teuthology.orchestra.run.smithi171.stderr:Warning: os-prober will not be executed to detect other bootable partitions.
2024-05-17T14:43:24.320 INFO:teuthology.orchestra.run.smithi171.stderr:Systems on them will not be added to the GRUB boot configuration.
2024-05-17T14:43:24.321 INFO:teuthology.orchestra.run.smithi171.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry.
2024-05-17T14:43:24.341 INFO:teuthology.orchestra.run.smithi171.stderr:done
2024-05-17T14:43:24.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo shutdown -r now
2024-05-17T14:43:28.850 INFO:teuthology.orchestra.run.smithi072.stdout:/etc/kernel/postinst.d/zz-update-grub:
2024-05-17T14:43:28.850 INFO:teuthology.orchestra.run.smithi072.stdout:Sourcing file `/etc/default/grub'
2024-05-17T14:43:28.881 INFO:teuthology.orchestra.run.smithi072.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg'
2024-05-17T14:43:28.904 INFO:teuthology.orchestra.run.smithi072.stdout:Generating grub configuration file ...
2024-05-17T14:43:29.183 INFO:teuthology.orchestra.run.smithi072.stdout:Found linux image: /boot/vmlinuz-5.15.0-107-generic
2024-05-17T14:43:29.280 INFO:teuthology.orchestra.run.smithi072.stdout:Found initrd image: /boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:29.576 INFO:teuthology.orchestra.run.smithi072.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic
2024-05-17T14:43:29.581 INFO:teuthology.orchestra.run.smithi072.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic
2024-05-17T14:43:29.664 INFO:teuthology.orchestra.run.smithi072.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic
2024-05-17T14:43:29.669 INFO:teuthology.orchestra.run.smithi072.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic
2024-05-17T14:43:29.977 INFO:teuthology.orchestra.run.smithi072.stdout:Warning: os-prober will not be executed to detect other bootable partitions.
2024-05-17T14:43:29.977 INFO:teuthology.orchestra.run.smithi072.stdout:Systems on them will not be added to the GRUB boot configuration.
2024-05-17T14:43:29.977 INFO:teuthology.orchestra.run.smithi072.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry.
2024-05-17T14:43:30.062 INFO:teuthology.orchestra.run.smithi072.stdout:done
2024-05-17T14:43:30.640 DEBUG:teuthology.orchestra.run.smithi072:> dpkg -s linux-image-generic
2024-05-17T14:43:30.665 INFO:teuthology.orchestra.run.smithi072.stdout:Package: linux-image-generic
2024-05-17T14:43:30.665 INFO:teuthology.orchestra.run.smithi072.stdout:Status: install ok installed
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Priority: optional
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Section: kernel
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Installed-Size: 21
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Maintainer: Ubuntu Kernel Team <kernel-team@lists.ubuntu.com>
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Architecture: amd64
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Source: linux-meta
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Version: 5.15.0.107.107
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-107), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3)
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Depends: linux-image-5.15.0-107-generic, linux-modules-extra-5.15.0-107-generic, linux-firmware, intel-microcode, amd64-microcode
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Recommends: thermald
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout:Description: Generic Linux kernel image
2024-05-17T14:43:30.666 INFO:teuthology.orchestra.run.smithi072.stdout: This package will always depend on the latest generic kernel image
2024-05-17T14:43:30.667 INFO:teuthology.orchestra.run.smithi072.stdout: available.
2024-05-17T14:43:30.667 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-107-generic
2024-05-17T14:43:30.667 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2024-05-17T14:43:30.667 DEBUG:teuthology.task.kernel:src is distro, skipping download
2024-05-17T14:43:30.667 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a...
2024-05-17T14:43:30.667 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi072.front.sepia.ceph.com, path=None, version=distro)
2024-05-17T14:43:30.667 DEBUG:teuthology.orchestra.run.smithi072:> sudo apt-get clean
2024-05-17T14:43:30.824 DEBUG:teuthology.orchestra.run.smithi072:> sudo apt-get update
2024-05-17T14:43:30.941 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease
2024-05-17T14:43:30.942 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease
2024-05-17T14:43:30.947 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease
2024-05-17T14:43:30.968 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease
2024-05-17T14:43:32.346 INFO:teuthology.orchestra.run.smithi072.stdout:Reading package lists...
2024-05-17T14:43:32.363 DEBUG:teuthology.orchestra.run.smithi072:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic
2024-05-17T14:43:32.424 INFO:teuthology.orchestra.run.smithi072.stdout:Reading package lists...
2024-05-17T14:43:32.626 INFO:teuthology.orchestra.run.smithi072.stdout:Building dependency tree...
2024-05-17T14:43:32.627 INFO:teuthology.orchestra.run.smithi072.stdout:Reading state information...
2024-05-17T14:43:32.794 INFO:teuthology.orchestra.run.smithi072.stdout:linux-image-generic is already the newest version (5.15.0.107.107).
2024-05-17T14:43:32.794 INFO:teuthology.orchestra.run.smithi072.stdout:The following packages were automatically installed and are no longer required:
2024-05-17T14:43:32.794 INFO:teuthology.orchestra.run.smithi072.stdout:  libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0
2024-05-17T14:43:32.794 INFO:teuthology.orchestra.run.smithi072.stdout:  libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib
2024-05-17T14:43:32.794 INFO:teuthology.orchestra.run.smithi072.stdout:  libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib
2024-05-17T14:43:32.795 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2
2024-05-17T14:43:32.795 INFO:teuthology.orchestra.run.smithi072.stdout:  python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal
2024-05-17T14:43:32.795 INFO:teuthology.orchestra.run.smithi072.stdout:Use 'sudo apt autoremove' to remove them.
2024-05-17T14:43:32.833 INFO:teuthology.orchestra.run.smithi072.stdout:0 upgraded, 0 newly installed, 0 to remove and 320 not upgraded.
2024-05-17T14:43:32.835 DEBUG:teuthology.orchestra.run.smithi072:> dpkg -s linux-image-generic
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Package: linux-image-generic
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Status: install ok installed
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Priority: optional
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Section: kernel
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Installed-Size: 21
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Maintainer: Ubuntu Kernel Team <kernel-team@lists.ubuntu.com>
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Architecture: amd64
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Source: linux-meta
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Version: 5.15.0.107.107
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-107), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3)
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Depends: linux-image-5.15.0-107-generic, linux-modules-extra-5.15.0-107-generic, linux-firmware, intel-microcode, amd64-microcode
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Recommends: thermald
2024-05-17T14:43:32.847 INFO:teuthology.orchestra.run.smithi072.stdout:Description: Generic Linux kernel image
2024-05-17T14:43:32.848 INFO:teuthology.orchestra.run.smithi072.stdout: This package will always depend on the latest generic kernel image
2024-05-17T14:43:32.848 INFO:teuthology.orchestra.run.smithi072.stdout: available.
2024-05-17T14:43:32.848 DEBUG:teuthology.orchestra.run.smithi072:> mktemp
2024-05-17T14:43:32.893 INFO:teuthology.orchestra.run.smithi072.stdout:/tmp/tmp.z5x2ayttD5
2024-05-17T14:43:32.894 DEBUG:teuthology.orchestra.run.smithi072:> sudo cp /boot/grub/grub.cfg /tmp/tmp.z5x2ayttD5
2024-05-17T14:43:32.954 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod 0666 /tmp/tmp.z5x2ayttD5
2024-05-17T14:43:33.092 DEBUG:teuthology.orchestra.remote:smithi072:/tmp/tmp.z5x2ayttD5 is 10KB
2024-05-17T14:43:33.104 DEBUG:teuthology.orchestra.run.smithi072:> rm -fr /tmp/tmp.z5x2ayttD5
2024-05-17T14:43:33.109 DEBUG:teuthology.orchestra.run.smithi072:> sudo rm -f -- /etc/grub.d/01_ceph_kernel
2024-05-17T14:43:33.165 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:43:33.165 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/etc/grub.d/01_ceph_kernel
2024-05-17T14:43:33.165 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod 755 /etc/grub.d/01_ceph_kernel
2024-05-17T14:43:33.268 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-107-generic
2024-05-17T14:43:33.268 DEBUG:teuthology.orchestra.run.smithi072:> sudo update-grub
2024-05-17T14:43:35.554 INFO:teuthology.orchestra.run.smithi072.stderr:Sourcing file `/etc/default/grub'
2024-05-17T14:43:35.563 INFO:teuthology.orchestra.run.smithi072.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg'
2024-05-17T14:43:35.577 INFO:teuthology.orchestra.run.smithi072.stderr:Generating grub configuration file ...
2024-05-17T14:43:35.815 INFO:teuthology.orchestra.run.smithi072.stderr:Found linux image: /boot/vmlinuz-5.15.0-107-generic
2024-05-17T14:43:35.820 INFO:teuthology.orchestra.run.smithi072.stderr:Found initrd image: /boot/initrd.img-5.15.0-107-generic
2024-05-17T14:43:36.058 INFO:teuthology.orchestra.run.smithi072.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic
2024-05-17T14:43:36.063 INFO:teuthology.orchestra.run.smithi072.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic
2024-05-17T14:43:36.100 INFO:teuthology.orchestra.run.smithi072.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic
2024-05-17T14:43:36.105 INFO:teuthology.orchestra.run.smithi072.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic
2024-05-17T14:43:36.222 INFO:teuthology.orchestra.run.smithi072.stderr:Warning: os-prober will not be executed to detect other bootable partitions.
2024-05-17T14:43:36.222 INFO:teuthology.orchestra.run.smithi072.stderr:Systems on them will not be added to the GRUB boot configuration.
2024-05-17T14:43:36.222 INFO:teuthology.orchestra.run.smithi072.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry.
2024-05-17T14:43:36.293 INFO:teuthology.orchestra.run.smithi072.stderr:done
2024-05-17T14:43:36.295 DEBUG:teuthology.orchestra.run.smithi072:> sudo shutdown -r now
2024-05-17T14:43:54.373 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version...
2024-05-17T14:43:54.374 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:43:54.375 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:44:06.301 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version...
2024-05-17T14:44:06.302 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:44:06.303 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi072.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:44:12.706 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.171
2024-05-17T14:44:21.715 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:44:21.716 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:44:24.738 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.72
2024-05-17T14:44:24.766 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.171
2024-05-17T14:44:33.746 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:44:33.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi072.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:44:36.767 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:44:36.768 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:44:36.798 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.72
2024-05-17T14:44:48.810 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:44:48.811 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi072.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:45:07.231 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.72
2024-05-17T14:45:08.255 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.171
2024-05-17T14:45:22.242 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:45:22.243 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi072.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:45:23.255 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:45:23.256 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:45:23.540 DEBUG:teuthology.orchestra.run.smithi072:> true
2024-05-17T14:45:23.662 DEBUG:teuthology.orchestra.run.smithi171:> true
2024-05-17T14:45:24.260 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:45:24.260 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.15.0-107-generic"...
2024-05-17T14:45:24.260 DEBUG:teuthology.orchestra.run.smithi171:> uname -r
2024-05-17T14:45:24.307 INFO:teuthology.orchestra.run.smithi171.stdout:5.15.0-107-generic
2024-05-17T14:45:24.308 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-107-generic vs 5.15.0-107-generic
2024-05-17T14:45:24.308 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2024-05-17T14:45:24.308 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu
2024-05-17T14:45:25.309 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a...
2024-05-17T14:45:25.310 DEBUG:teuthology.orchestra.run.smithi171:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-05-17T14:45:25.373 INFO:teuthology.orchestra.run.smithi171.stdout:ttyS1
2024-05-17T14:45:25.397 DEBUG:teuthology.parallel:result is None
2024-05-17T14:45:25.533 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:45:25.533 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.15.0-107-generic"...
2024-05-17T14:45:25.534 DEBUG:teuthology.orchestra.run.smithi072:> uname -r
2024-05-17T14:45:25.539 INFO:teuthology.orchestra.run.smithi072.stdout:5.15.0-107-generic
2024-05-17T14:45:25.539 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-107-generic vs 5.15.0-107-generic
2024-05-17T14:45:25.539 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2024-05-17T14:45:25.539 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu
2024-05-17T14:45:26.540 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a...
2024-05-17T14:45:26.540 DEBUG:teuthology.orchestra.run.smithi072:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-05-17T14:45:26.892 INFO:teuthology.orchestra.run.smithi072.stdout:ttyS1
2024-05-17T14:45:26.917 DEBUG:teuthology.parallel:result is None
2024-05-17T14:45:26.917 INFO:teuthology.run_tasks:Running task internal.base...
2024-05-17T14:45:26.925 INFO:teuthology.task.internal:Creating test directory...
2024-05-17T14:45:26.925 DEBUG:teuthology.orchestra.run.smithi072:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-05-17T14:45:26.928 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-05-17T14:45:26.934 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2024-05-17T14:45:26.957 INFO:teuthology.run_tasks:Running task internal.archive...
2024-05-17T14:45:26.967 INFO:teuthology.task.internal:Creating archive directory...
2024-05-17T14:45:26.968 DEBUG:teuthology.orchestra.run.smithi072:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-05-17T14:45:26.975 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-05-17T14:45:27.071 INFO:teuthology.run_tasks:Running task internal.coredump...
2024-05-17T14:45:27.078 INFO:teuthology.task.internal:Enabling coredump saving...
2024-05-17T14:45:27.078 DEBUG:teuthology.orchestra.run.smithi072:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-05-17T14:45:27.081 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-05-17T14:45:27.102 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-05-17T14:45:27.112 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-05-17T14:45:27.116 INFO:teuthology.orchestra.run.smithi072.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-05-17T14:45:27.123 INFO:teuthology.orchestra.run.smithi072.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-05-17T14:45:27.125 INFO:teuthology.run_tasks:Running task internal.sudo...
2024-05-17T14:45:27.146 INFO:teuthology.task.internal:Configuring sudo...
2024-05-17T14:45:27.146 DEBUG:teuthology.orchestra.run.smithi072:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-05-17T14:45:27.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-05-17T14:45:27.192 INFO:teuthology.run_tasks:Running task internal.syslog...
2024-05-17T14:45:27.201 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2024-05-17T14:45:27.201 DEBUG:teuthology.orchestra.run.smithi072:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-05-17T14:45:27.224 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-05-17T14:45:27.240 DEBUG:teuthology.orchestra.run.smithi072:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-05-17T14:45:27.275 DEBUG:teuthology.orchestra.run.smithi072:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-05-17T14:45:27.324 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:45:27.324 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-05-17T14:45:27.400 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-05-17T14:45:27.406 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-05-17T14:45:27.451 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:45:27.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-05-17T14:45:27.510 DEBUG:teuthology.orchestra.run.smithi072:> sudo service rsyslog restart
2024-05-17T14:45:27.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo service rsyslog restart
2024-05-17T14:45:27.613 INFO:teuthology.run_tasks:Running task internal.timer...
2024-05-17T14:45:27.620 INFO:teuthology.task.internal:Starting timer...
2024-05-17T14:45:27.620 INFO:teuthology.run_tasks:Running task pcp...
2024-05-17T14:45:27.631 INFO:teuthology.run_tasks:Running task selinux...
2024-05-17T14:45:27.639 DEBUG:teuthology.task.selinux:Excluding smithi072: OS 'ubuntu' does not support SELinux
2024-05-17T14:45:27.639 DEBUG:teuthology.task.selinux:Excluding smithi171: OS 'ubuntu' does not support SELinux
2024-05-17T14:45:27.640 DEBUG:teuthology.task.selinux:Getting current SELinux state
2024-05-17T14:45:27.640 DEBUG:teuthology.task.selinux:Existing SELinux modes: {}
2024-05-17T14:45:27.640 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2024-05-17T14:45:27.640 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2024-05-17T14:45:27.648 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2024-05-17T14:45:27.668 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2024-05-17T14:45:27.669 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi072.front.sepia.ceph.com,smithi171.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2024-05-17T14:52:36.136 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi072.front.sepia.ceph.com'), Remote(name='ubuntu@smithi171.front.sepia.ceph.com')]
2024-05-17T14:52:36.138 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:52:36.143 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi072.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:52:36.229 DEBUG:teuthology.orchestra.run.smithi072:> true
2024-05-17T14:52:36.310 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi072.front.sepia.ceph.com'
2024-05-17T14:52:36.311 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:52:36.311 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-05-17T14:52:36.392 DEBUG:teuthology.orchestra.run.smithi171:> true
2024-05-17T14:52:36.470 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi171.front.sepia.ceph.com'
2024-05-17T14:52:36.471 INFO:teuthology.run_tasks:Running task clock...
2024-05-17T14:52:36.482 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew...
2024-05-17T14:52:36.482 INFO:teuthology.orchestra.run:Running command with timeout 360
2024-05-17T14:52:36.482 DEBUG:teuthology.orchestra.run.smithi072:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
2024-05-17T14:52:36.486 INFO:teuthology.orchestra.run:Running command with timeout 360
2024-05-17T14:52:36.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
2024-05-17T14:52:36.512 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Command line: ntpd -gq
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: ----------------------------------------------------
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: ntp-4 is maintained by Network Time Foundation,
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Inc. (NTF), a non-profit 501(c)(3) public-benefit
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: corporation.  Support and training for ntp-4 are
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: available at https://www.nwtime.org/support
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: ----------------------------------------------------
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: proto: precision = 0.059 usec (-24)
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: basedate set to 2022-02-04
2024-05-17T14:52:36.513 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: gps base set to 2022-02-06 (week 2196)
2024-05-17T14:52:36.514 INFO:teuthology.orchestra.run.smithi072.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED.
2024-05-17T14:52:36.514 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: restrict 0.0.0.0: KOD does nothing without LIMITED.
2024-05-17T14:52:36.514 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: restrict ::: KOD does nothing without LIMITED.
2024-05-17T14:52:36.514 INFO:teuthology.orchestra.run.smithi072.stderr:restrict ::: KOD does nothing without LIMITED.
2024-05-17T14:52:36.515 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listen and drop on 0 v6wildcard [::]:123
2024-05-17T14:52:36.515 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listen and drop on 1 v4wildcard 0.0.0.0:123
2024-05-17T14:52:36.515 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listen normally on 2 lo 127.0.0.1:123
2024-05-17T14:52:36.515 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listen normally on 3 enp3s0f1 172.21.15.72:123
2024-05-17T14:52:36.516 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listen normally on 4 lo [::1]:123
2024-05-17T14:52:36.516 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:b765%5]:123
2024-05-17T14:52:36.516 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:36 ntpd[17310]: Listening on routing socket on fd #22 for interface updates
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Command line: ntpd -gq
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: ----------------------------------------------------
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: ntp-4 is maintained by Network Time Foundation,
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Inc. (NTF), a non-profit 501(c)(3) public-benefit
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: corporation.  Support and training for ntp-4 are
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: available at https://www.nwtime.org/support
2024-05-17T14:52:36.538 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: ----------------------------------------------------
2024-05-17T14:52:36.539 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: proto: precision = 0.059 usec (-24)
2024-05-17T14:52:36.539 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: basedate set to 2022-02-04
2024-05-17T14:52:36.539 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: gps base set to 2022-02-06 (week 2196)
2024-05-17T14:52:36.540 INFO:teuthology.orchestra.run.smithi171.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED.
2024-05-17T14:52:36.540 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: restrict 0.0.0.0: KOD does nothing without LIMITED.
2024-05-17T14:52:36.540 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: restrict ::: KOD does nothing without LIMITED.
2024-05-17T14:52:36.540 INFO:teuthology.orchestra.run.smithi171.stderr:restrict ::: KOD does nothing without LIMITED.
2024-05-17T14:52:36.541 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listen and drop on 0 v6wildcard [::]:123
2024-05-17T14:52:36.541 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listen and drop on 1 v4wildcard 0.0.0.0:123
2024-05-17T14:52:36.541 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listen normally on 2 lo 127.0.0.1:123
2024-05-17T14:52:36.541 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listen normally on 3 enp3s0f1 172.21.15.171:123
2024-05-17T14:52:36.541 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listen normally on 4 lo [::1]:123
2024-05-17T14:52:36.542 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6f6d%5]:123
2024-05-17T14:52:36.542 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:36 ntpd[17253]: Listening on routing socket on fd #22 for interface updates
2024-05-17T14:52:37.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:37.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:37.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:37.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:37.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:37 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:38.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:38 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:38.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:38 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:38.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:38 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:38.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:38 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:38.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:38 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:38.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:38 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:39.512 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:39 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.512 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:39 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:39.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:39 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:40.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:40.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:40.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:40.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:40.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:40.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:40.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:40 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:41.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:41.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:41.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:41.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:41.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:41.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:41.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:41 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:42.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:42.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:42.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:42.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:42.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:42.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:42.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:42.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:42 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:43.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:43.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:43.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:43.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:43.539 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stdout:17 May 14:52:43 ntpd[17253]: ntpd: time slew -0.001708 s
2024-05-17T14:52:43.540 INFO:teuthology.orchestra.run.smithi171.stdout:ntpd: time slew -0.001708s
2024-05-17T14:52:43.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied
2024-05-17T14:52:43.541 INFO:teuthology.orchestra.run.smithi171.stderr:17 May 14:52:43 ntpd[17253]: can't open /var/log/ntpstats/loopstats.20240517: Permission denied
2024-05-17T14:52:43.594 INFO:teuthology.orchestra.run.smithi171.stdout:     remote           refid      st t when poll reach   delay   offset  jitter
2024-05-17T14:52:43.594 INFO:teuthology.orchestra.run.smithi171.stdout:==============================================================================
2024-05-17T14:52:43.594 INFO:teuthology.orchestra.run.smithi171.stdout: hv01.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:43.594 INFO:teuthology.orchestra.run.smithi171.stdout: hv02.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:43.594 INFO:teuthology.orchestra.run.smithi171.stdout: hv03.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:43.595 INFO:teuthology.orchestra.run.smithi171.stdout: hv04.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:44.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:44.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:44.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:44.513 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/rawstats.20240517: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/peerstats.20240517: Permission denied
2024-05-17T14:52:44.514 INFO:teuthology.orchestra.run.smithi072.stdout:17 May 14:52:44 ntpd[17310]: ntpd: time slew +0.000300 s
2024-05-17T14:52:44.515 INFO:teuthology.orchestra.run.smithi072.stdout:ntpd: time slew +0.000300s
2024-05-17T14:52:44.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied
2024-05-17T14:52:44.515 INFO:teuthology.orchestra.run.smithi072.stderr:17 May 14:52:44 ntpd[17310]: can't open /var/log/ntpstats/loopstats.20240517: Permission denied
2024-05-17T14:52:44.558 INFO:teuthology.orchestra.run.smithi072.stdout:     remote           refid      st t when poll reach   delay   offset  jitter
2024-05-17T14:52:44.559 INFO:teuthology.orchestra.run.smithi072.stdout:==============================================================================
2024-05-17T14:52:44.559 INFO:teuthology.orchestra.run.smithi072.stdout: hv01.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:44.559 INFO:teuthology.orchestra.run.smithi072.stdout: hv02.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:44.559 INFO:teuthology.orchestra.run.smithi072.stdout: hv03.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:44.559 INFO:teuthology.orchestra.run.smithi072.stdout: hv04.front.sepi .INIT.          16 u    -   64    0    0.000   +0.000   0.000
2024-05-17T14:52:44.560 INFO:teuthology.run_tasks:Running task install...
2024-05-17T14:52:44.568 DEBUG:teuthology.task.install:project ceph
2024-05-17T14:52:44.568 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}
2024-05-17T14:52:44.569 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}
2024-05-17T14:52:44.569 INFO:teuthology.task.install:Using flavor: default
2024-05-17T14:52:44.573 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']}
2024-05-17T14:52:44.573 INFO:teuthology.task.install:extra packages: ['rbd-mirror']
2024-05-17T14:52:44.574 DEBUG:teuthology.orchestra.run.smithi072:> sudo apt-key list | grep Ceph
2024-05-17T14:52:44.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo apt-key list | grep Ceph
2024-05-17T14:52:44.641 INFO:teuthology.orchestra.run.smithi171.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).
2024-05-17T14:52:44.656 INFO:teuthology.orchestra.run.smithi072.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).
2024-05-17T14:52:44.673 INFO:teuthology.orchestra.run.smithi171.stdout:uid           [ unknown] Ceph automated package build (Ceph automated package build) <sage@newdream.net>
2024-05-17T14:52:44.673 INFO:teuthology.orchestra.run.smithi171.stdout:uid           [ unknown] Ceph.com (release key) <security@ceph.com>
2024-05-17T14:52:44.674 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64
2024-05-17T14:52:44.674 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:52:44.683 INFO:teuthology.orchestra.run.smithi072.stdout:uid           [ unknown] Ceph automated package build (Ceph automated package build) <sage@newdream.net>
2024-05-17T14:52:44.683 INFO:teuthology.orchestra.run.smithi072.stdout:uid           [ unknown] Ceph.com (release key) <security@ceph.com>
2024-05-17T14:52:44.684 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64
2024-05-17T14:52:44.684 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:52:44.822 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default/
2024-05-17T14:52:44.822 INFO:teuthology.task.install.deb:Package version is 19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:52:44.936 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:52:44.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/apt/sources.list.d/ceph.list
2024-05-17T14:52:44.951 DEBUG:teuthology.orchestra.run.smithi171:> sudo apt-get update
2024-05-17T14:52:44.957 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default/
2024-05-17T14:52:44.957 INFO:teuthology.task.install.deb:Package version is 19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:52:45.068 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:52:45.068 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/etc/apt/sources.list.d/ceph.list
2024-05-17T14:52:45.083 DEBUG:teuthology.orchestra.run.smithi072:> sudo apt-get update
2024-05-17T14:52:45.198 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease
2024-05-17T14:52:45.234 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease
2024-05-17T14:52:45.300 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease
2024-05-17T14:52:45.300 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease
2024-05-17T14:52:45.315 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease
2024-05-17T14:52:45.332 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease
2024-05-17T14:52:45.334 INFO:teuthology.orchestra.run.smithi171.stdout:Ign:4 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy InRelease
2024-05-17T14:52:45.383 INFO:teuthology.orchestra.run.smithi072.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease
2024-05-17T14:52:45.397 INFO:teuthology.orchestra.run.smithi171.stdout:Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease
2024-05-17T14:52:45.397 INFO:teuthology.orchestra.run.smithi171.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy Release [6,978 B]
2024-05-17T14:52:45.448 INFO:teuthology.orchestra.run.smithi072.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy InRelease
2024-05-17T14:52:45.449 INFO:teuthology.orchestra.run.smithi171.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy Release.gpg
2024-05-17T14:52:45.516 INFO:teuthology.orchestra.run.smithi072.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy Release [6,978 B]
2024-05-17T14:52:45.517 INFO:teuthology.orchestra.run.smithi171.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,415 B]
2024-05-17T14:52:45.587 INFO:teuthology.orchestra.run.smithi072.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy Release.gpg
2024-05-17T14:52:45.595 INFO:teuthology.orchestra.run.smithi171.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.1 kB]
2024-05-17T14:52:45.655 INFO:teuthology.orchestra.run.smithi072.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.1 kB]
2024-05-17T14:52:45.677 INFO:teuthology.orchestra.run.smithi171.stdout:Fetched 28.5 kB in 1s (52.3 kB/s)
2024-05-17T14:52:45.710 INFO:teuthology.orchestra.run.smithi072.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,415 B]
2024-05-17T14:52:45.793 INFO:teuthology.orchestra.run.smithi072.stdout:Fetched 28.5 kB in 1s (54.6 kB/s)
2024-05-17T14:52:46.656 INFO:teuthology.orchestra.run.smithi171.stdout:Reading package lists...
2024-05-17T14:52:46.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.0.0-3735-gee59d86a-1jammy cephadm=19.0.0-3735-gee59d86a-1jammy ceph-mds=19.0.0-3735-gee59d86a-1jammy ceph-mgr=19.0.0-3735-gee59d86a-1jammy ceph-common=19.0.0-3735-gee59d86a-1jammy ceph-fuse=19.0.0-3735-gee59d86a-1jammy ceph-test=19.0.0-3735-gee59d86a-1jammy ceph-volume=19.0.0-3735-gee59d86a-1jammy radosgw=19.0.0-3735-gee59d86a-1jammy python3-rados=19.0.0-3735-gee59d86a-1jammy python3-rgw=19.0.0-3735-gee59d86a-1jammy python3-cephfs=19.0.0-3735-gee59d86a-1jammy python3-rbd=19.0.0-3735-gee59d86a-1jammy libcephfs2=19.0.0-3735-gee59d86a-1jammy libcephfs-dev=19.0.0-3735-gee59d86a-1jammy librados2=19.0.0-3735-gee59d86a-1jammy librbd1=19.0.0-3735-gee59d86a-1jammy rbd-fuse=19.0.0-3735-gee59d86a-1jammy rbd-mirror=19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:52:46.735 INFO:teuthology.orchestra.run.smithi171.stdout:Reading package lists...
2024-05-17T14:52:46.753 INFO:teuthology.orchestra.run.smithi072.stdout:Reading package lists...
2024-05-17T14:52:46.768 DEBUG:teuthology.orchestra.run.smithi072:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.0.0-3735-gee59d86a-1jammy cephadm=19.0.0-3735-gee59d86a-1jammy ceph-mds=19.0.0-3735-gee59d86a-1jammy ceph-mgr=19.0.0-3735-gee59d86a-1jammy ceph-common=19.0.0-3735-gee59d86a-1jammy ceph-fuse=19.0.0-3735-gee59d86a-1jammy ceph-test=19.0.0-3735-gee59d86a-1jammy ceph-volume=19.0.0-3735-gee59d86a-1jammy radosgw=19.0.0-3735-gee59d86a-1jammy python3-rados=19.0.0-3735-gee59d86a-1jammy python3-rgw=19.0.0-3735-gee59d86a-1jammy python3-cephfs=19.0.0-3735-gee59d86a-1jammy python3-rbd=19.0.0-3735-gee59d86a-1jammy libcephfs2=19.0.0-3735-gee59d86a-1jammy libcephfs-dev=19.0.0-3735-gee59d86a-1jammy librados2=19.0.0-3735-gee59d86a-1jammy librbd1=19.0.0-3735-gee59d86a-1jammy rbd-fuse=19.0.0-3735-gee59d86a-1jammy rbd-mirror=19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:52:46.830 INFO:teuthology.orchestra.run.smithi072.stdout:Reading package lists...
2024-05-17T14:52:46.937 INFO:teuthology.orchestra.run.smithi171.stdout:Building dependency tree...
2024-05-17T14:52:46.937 INFO:teuthology.orchestra.run.smithi171.stdout:Reading state information...
2024-05-17T14:52:47.040 INFO:teuthology.orchestra.run.smithi072.stdout:Building dependency tree...
2024-05-17T14:52:47.041 INFO:teuthology.orchestra.run.smithi072.stdout:Reading state information...
2024-05-17T14:52:47.137 INFO:teuthology.orchestra.run.smithi171.stdout:The following packages were automatically installed and are no longer required:
2024-05-17T14:52:47.137 INFO:teuthology.orchestra.run.smithi171.stdout:  libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0
2024-05-17T14:52:47.137 INFO:teuthology.orchestra.run.smithi171.stdout:  libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib
2024-05-17T14:52:47.137 INFO:teuthology.orchestra.run.smithi171.stdout:  libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib
2024-05-17T14:52:47.138 INFO:teuthology.orchestra.run.smithi171.stdout:  linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2
2024-05-17T14:52:47.138 INFO:teuthology.orchestra.run.smithi171.stdout:  python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal
2024-05-17T14:52:47.138 INFO:teuthology.orchestra.run.smithi171.stdout:Use 'sudo apt autoremove' to remove them.
2024-05-17T14:52:47.139 INFO:teuthology.orchestra.run.smithi171.stdout:The following additional packages will be installed:
2024-05-17T14:52:47.139 INFO:teuthology.orchestra.run.smithi171.stdout:  ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local
2024-05-17T14:52:47.139 INFO:teuthology.orchestra.run.smithi171.stdout:  ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq
2024-05-17T14:52:47.139 INFO:teuthology.orchestra.run.smithi171.stdout:  libdouble-conversion3 libjq1 liblttng-ust1 liblua5.3-dev liboath0 libonig5
2024-05-17T14:52:47.139 INFO:teuthology.orchestra.run.smithi171.stdout:  libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1
2024-05-17T14:52:47.140 INFO:teuthology.orchestra.run.smithi171.stdout:  librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0
2024-05-17T14:52:47.140 INFO:teuthology.orchestra.run.smithi171.stdout:  lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli pkg-config
2024-05-17T14:52:47.140 INFO:teuthology.orchestra.run.smithi171.stdout:  python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl
2024-05-17T14:52:47.140 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-asyncssh python3-babel python3-bcrypt python3-cachetools
2024-05-17T14:52:47.140 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot
2024-05-17T14:52:47.140 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig
2024-05-17T14:52:47.141 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools
2024-05-17T14:52:47.141 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes
2024-05-17T14:52:47.141 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-logutils python3-mako python3-markupsafe python3-natsort
2024-05-17T14:52:47.141 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-openssl python3-paste python3-pastedeploy python3-pastescript
2024-05-17T14:52:47.141 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-pecan python3-pluggy python3-portend python3-prettytable
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-pygments python3-pyinotify python3-pytest python3-repoze.lru
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-requests python3-requests-oauthlib python3-routes python3-rsa
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-simplegeneric python3-simplejson python3-singledispatch
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-sklearn python3-sklearn-lib python3-tempita python3-tempora
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-threadpoolctl python3-toml python3-urllib3 python3-waitress
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-wcwidth python3-webob python3-websocket python3-webtest
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat unzip
2024-05-17T14:52:47.142 INFO:teuthology.orchestra.run.smithi171.stdout:  xmlstarlet zip
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:Suggested packages:
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-influxdb readline-doc python-jinja2-doc python3-beaker
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  python-mako-doc python-natsort-doc python-openssl-doc python3-openssl-dbg
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  python-pecan-doc python-psutil-doc subversion python-pygments-doc
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  python-pyinotify-doc python3-socks python-requests-doc python3-dap
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  python-sklearn-doc ipython3 python-waitress-doc python-webob-doc
2024-05-17T14:52:47.143 INFO:teuthology.orchestra.run.smithi171.stdout:  python-webtest-doc python-werkzeug-doc python3-watchdog
2024-05-17T14:52:47.144 INFO:teuthology.orchestra.run.smithi171.stdout:Recommended packages:
2024-05-17T14:52:47.144 INFO:teuthology.orchestra.run.smithi171.stdout:  btrfs-tools
2024-05-17T14:52:47.241 INFO:teuthology.orchestra.run.smithi072.stdout:The following packages were automatically installed and are no longer required:
2024-05-17T14:52:47.241 INFO:teuthology.orchestra.run.smithi072.stdout:  libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0
2024-05-17T14:52:47.241 INFO:teuthology.orchestra.run.smithi072.stdout:  libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib
2024-05-17T14:52:47.241 INFO:teuthology.orchestra.run.smithi072.stdout:  libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib
2024-05-17T14:52:47.243 INFO:teuthology.orchestra.run.smithi072.stdout:  linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2
2024-05-17T14:52:47.243 INFO:teuthology.orchestra.run.smithi072.stdout:  python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal
2024-05-17T14:52:47.243 INFO:teuthology.orchestra.run.smithi072.stdout:Use 'sudo apt autoremove' to remove them.
2024-05-17T14:52:47.243 INFO:teuthology.orchestra.run.smithi072.stdout:The following additional packages will be installed:
2024-05-17T14:52:47.243 INFO:teuthology.orchestra.run.smithi072.stdout:  ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local
2024-05-17T14:52:47.243 INFO:teuthology.orchestra.run.smithi072.stdout:  ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq
2024-05-17T14:52:47.244 INFO:teuthology.orchestra.run.smithi171.stdout:The following NEW packages will be installed:
2024-05-17T14:52:47.244 INFO:teuthology.orchestra.run.smithi171.stdout:  ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 liblttng-ust1
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  liblua5.3-dev liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  python-babel-localedata python-pastedeploy-tpl python3-asyncssh
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-ceph-common python3-cephfs python3-certifi python3-cheroot
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig
2024-05-17T14:52:47.245 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-logutils python3-mako python3-markupsafe python3-natsort
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-openssl python3-paste python3-pastedeploy python3-pastescript
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-pecan python3-pluggy python3-portend python3-prettytable
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-routes python3-rsa python3-simplegeneric python3-simplejson
2024-05-17T14:52:47.246 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita
2024-05-17T14:52:47.247 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-tempora python3-threadpoolctl python3-toml python3-urllib3
2024-05-17T14:52:47.247 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-waitress python3-wcwidth python3-webob python3-websocket
2024-05-17T14:52:47.247 INFO:teuthology.orchestra.run.smithi171.stdout:  python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n
2024-05-17T14:52:47.247 INFO:teuthology.orchestra.run.smithi171.stdout:  radosgw rbd-fuse rbd-mirror socat unzip xmlstarlet zip
2024-05-17T14:52:47.248 INFO:teuthology.orchestra.run.smithi072.stdout:  libdouble-conversion3 libjq1 liblttng-ust1 liblua5.3-dev liboath0 libonig5
2024-05-17T14:52:47.248 INFO:teuthology.orchestra.run.smithi072.stdout:  libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1
2024-05-17T14:52:47.248 INFO:teuthology.orchestra.run.smithi072.stdout:  librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli pkg-config
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-asyncssh python3-babel python3-bcrypt python3-cachetools
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-logutils python3-mako python3-markupsafe python3-natsort
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-openssl python3-paste python3-pastedeploy python3-pastescript
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-pecan python3-pluggy python3-portend python3-prettytable
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules
2024-05-17T14:52:47.249 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-pygments python3-pyinotify python3-pytest python3-repoze.lru
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-requests python3-requests-oauthlib python3-routes python3-rsa
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-simplegeneric python3-simplejson python3-singledispatch
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-sklearn python3-sklearn-lib python3-tempita python3-tempora
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-threadpoolctl python3-toml python3-urllib3 python3-waitress
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-wcwidth python3-webob python3-websocket python3-webtest
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat unzip
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  xmlstarlet zip
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:Suggested packages:
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-influxdb readline-doc python-jinja2-doc python3-beaker
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python-mako-doc python-natsort-doc python-openssl-doc python3-openssl-dbg
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit
2024-05-17T14:52:47.250 INFO:teuthology.orchestra.run.smithi072.stdout:  python-pecan-doc python-psutil-doc subversion python-pygments-doc
2024-05-17T14:52:47.251 INFO:teuthology.orchestra.run.smithi072.stdout:  python-pyinotify-doc python3-socks python-requests-doc python3-dap
2024-05-17T14:52:47.251 INFO:teuthology.orchestra.run.smithi072.stdout:  python-sklearn-doc ipython3 python-waitress-doc python-webob-doc
2024-05-17T14:52:47.251 INFO:teuthology.orchestra.run.smithi072.stdout:  python-webtest-doc python-werkzeug-doc python3-watchdog
2024-05-17T14:52:47.251 INFO:teuthology.orchestra.run.smithi072.stdout:Recommended packages:
2024-05-17T14:52:47.252 INFO:teuthology.orchestra.run.smithi072.stdout:  btrfs-tools
2024-05-17T14:52:47.333 INFO:teuthology.orchestra.run.smithi171.stdout:0 upgraded, 119 newly installed, 0 to remove and 307 not upgraded.
2024-05-17T14:52:47.333 INFO:teuthology.orchestra.run.smithi171.stdout:Need to get 192 MB of archives.
2024-05-17T14:52:47.333 INFO:teuthology.orchestra.run.smithi171.stdout:After this operation, 875 MB of additional disk space will be used.
2024-05-17T14:52:47.334 INFO:teuthology.orchestra.run.smithi171.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB]
2024-05-17T14:52:47.345 INFO:teuthology.orchestra.run.smithi072.stdout:The following NEW packages will be installed:
2024-05-17T14:52:47.346 INFO:teuthology.orchestra.run.smithi072.stdout:  ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm
2024-05-17T14:52:47.346 INFO:teuthology.orchestra.run.smithi072.stdout:  ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents
2024-05-17T14:52:47.346 INFO:teuthology.orchestra.run.smithi072.stdout:  ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq
2024-05-17T14:52:47.346 INFO:teuthology.orchestra.run.smithi072.stdout:  libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 liblttng-ust1
2024-05-17T14:52:47.346 INFO:teuthology.orchestra.run.smithi072.stdout:  liblua5.3-dev liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5
2024-05-17T14:52:47.346 INFO:teuthology.orchestra.run.smithi072.stdout:  libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1
2024-05-17T14:52:47.347 INFO:teuthology.orchestra.run.smithi072.stdout:  libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec
2024-05-17T14:52:47.347 INFO:teuthology.orchestra.run.smithi072.stdout:  lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc
2024-05-17T14:52:47.347 INFO:teuthology.orchestra.run.smithi072.stdout:  python-babel-localedata python-pastedeploy-tpl python3-asyncssh
2024-05-17T14:52:47.347 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse
2024-05-17T14:52:47.347 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-ceph-common python3-cephfs python3-certifi python3-cheroot
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-logutils python3-mako python3-markupsafe python3-natsort
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-openssl python3-paste python3-pastedeploy python3-pastescript
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-pecan python3-pluggy python3-portend python3-prettytable
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-routes python3-rsa python3-simplegeneric python3-simplejson
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-tempora python3-threadpoolctl python3-toml python3-urllib3
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-waitress python3-wcwidth python3-webob python3-websocket
2024-05-17T14:52:47.348 INFO:teuthology.orchestra.run.smithi072.stdout:  python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n
2024-05-17T14:52:47.349 INFO:teuthology.orchestra.run.smithi072.stdout:  radosgw rbd-fuse rbd-mirror socat unzip xmlstarlet zip
2024-05-17T14:52:47.516 INFO:teuthology.orchestra.run.smithi171.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB]
2024-05-17T14:52:47.519 INFO:teuthology.orchestra.run.smithi171.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB]
2024-05-17T14:52:47.533 INFO:teuthology.orchestra.run.smithi072.stdout:0 upgraded, 119 newly installed, 0 to remove and 307 not upgraded.
2024-05-17T14:52:47.533 INFO:teuthology.orchestra.run.smithi072.stdout:Need to get 192 MB of archives.
2024-05-17T14:52:47.534 INFO:teuthology.orchestra.run.smithi072.stdout:After this operation, 875 MB of additional disk space will be used.
2024-05-17T14:52:47.534 INFO:teuthology.orchestra.run.smithi072.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB]
2024-05-17T14:52:47.551 INFO:teuthology.orchestra.run.smithi171.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB]
2024-05-17T14:52:47.599 INFO:teuthology.orchestra.run.smithi171.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.0.0-3735-gee59d86a-1jammy [3,557 kB]
2024-05-17T14:52:47.651 INFO:teuthology.orchestra.run.smithi171.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB]
2024-05-17T14:52:47.658 INFO:teuthology.orchestra.run.smithi171.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB]
2024-05-17T14:52:47.676 INFO:teuthology.orchestra.run.smithi171.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB]
2024-05-17T14:52:47.681 INFO:teuthology.orchestra.run.smithi171.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB]
2024-05-17T14:52:47.682 INFO:teuthology.orchestra.run.smithi171.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB]
2024-05-17T14:52:47.683 INFO:teuthology.orchestra.run.smithi171.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB]
2024-05-17T14:52:47.694 INFO:teuthology.orchestra.run.smithi171.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB]
2024-05-17T14:52:47.700 INFO:teuthology.orchestra.run.smithi171.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB]
2024-05-17T14:52:47.703 INFO:teuthology.orchestra.run.smithi171.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB]
2024-05-17T14:52:47.706 INFO:teuthology.orchestra.run.smithi171.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5,034 B]
2024-05-17T14:52:47.735 INFO:teuthology.orchestra.run.smithi171.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB]
2024-05-17T14:52:47.739 INFO:teuthology.orchestra.run.smithi171.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB]
2024-05-17T14:52:47.743 INFO:teuthology.orchestra.run.smithi171.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB]
2024-05-17T14:52:47.746 INFO:teuthology.orchestra.run.smithi171.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB]
2024-05-17T14:52:47.747 INFO:teuthology.orchestra.run.smithi171.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B]
2024-05-17T14:52:47.747 INFO:teuthology.orchestra.run.smithi171.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB]
2024-05-17T14:52:47.749 INFO:teuthology.orchestra.run.smithi171.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B]
2024-05-17T14:52:47.750 INFO:teuthology.orchestra.run.smithi171.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B]
2024-05-17T14:52:47.772 INFO:teuthology.orchestra.run.smithi171.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB]
2024-05-17T14:52:47.772 INFO:teuthology.orchestra.run.smithi171.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB]
2024-05-17T14:52:47.794 INFO:teuthology.orchestra.run.smithi171.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B]
2024-05-17T14:52:47.794 INFO:teuthology.orchestra.run.smithi171.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B]
2024-05-17T14:52:47.795 INFO:teuthology.orchestra.run.smithi171.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB]
2024-05-17T14:52:47.802 INFO:teuthology.orchestra.run.smithi171.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB]
2024-05-17T14:52:47.802 INFO:teuthology.orchestra.run.smithi171.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB]
2024-05-17T14:52:47.803 INFO:teuthology.orchestra.run.smithi171.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB]
2024-05-17T14:52:47.803 INFO:teuthology.orchestra.run.smithi171.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB]
2024-05-17T14:52:47.805 INFO:teuthology.orchestra.run.smithi171.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB]
2024-05-17T14:52:47.814 INFO:teuthology.orchestra.run.smithi171.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B]
2024-05-17T14:52:47.814 INFO:teuthology.orchestra.run.smithi171.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB]
2024-05-17T14:52:47.834 INFO:teuthology.orchestra.run.smithi171.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB]
2024-05-17T14:52:47.836 INFO:teuthology.orchestra.run.smithi171.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB]
2024-05-17T14:52:47.836 INFO:teuthology.orchestra.run.smithi171.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB]
2024-05-17T14:52:47.847 INFO:teuthology.orchestra.run.smithi171.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B]
2024-05-17T14:52:47.848 INFO:teuthology.orchestra.run.smithi171.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB]
2024-05-17T14:52:47.849 INFO:teuthology.orchestra.run.smithi171.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB]
2024-05-17T14:52:47.850 INFO:teuthology.orchestra.run.smithi171.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB]
2024-05-17T14:52:47.852 INFO:teuthology.orchestra.run.smithi171.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB]
2024-05-17T14:52:47.862 INFO:teuthology.orchestra.run.smithi171.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB]
2024-05-17T14:52:47.864 INFO:teuthology.orchestra.run.smithi171.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB]
2024-05-17T14:52:47.865 INFO:teuthology.orchestra.run.smithi072.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.0.0-3735-gee59d86a-1jammy [3,557 kB]
2024-05-17T14:52:47.889 INFO:teuthology.orchestra.run.smithi171.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB]
2024-05-17T14:52:47.890 INFO:teuthology.orchestra.run.smithi171.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB]
2024-05-17T14:52:47.895 INFO:teuthology.orchestra.run.smithi171.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB]
2024-05-17T14:52:47.896 INFO:teuthology.orchestra.run.smithi171.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB]
2024-05-17T14:52:47.898 INFO:teuthology.orchestra.run.smithi171.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB]
2024-05-17T14:52:47.902 INFO:teuthology.orchestra.run.smithi171.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB]
2024-05-17T14:52:47.955 INFO:teuthology.orchestra.run.smithi072.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB]
2024-05-17T14:52:47.967 INFO:teuthology.orchestra.run.smithi072.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB]
2024-05-17T14:52:48.049 INFO:teuthology.orchestra.run.smithi072.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB]
2024-05-17T14:52:48.060 INFO:teuthology.orchestra.run.smithi171.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB]
2024-05-17T14:52:48.062 INFO:teuthology.orchestra.run.smithi171.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB]
2024-05-17T14:52:48.064 INFO:teuthology.orchestra.run.smithi171.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB]
2024-05-17T14:52:48.065 INFO:teuthology.orchestra.run.smithi171.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB]
2024-05-17T14:52:48.067 INFO:teuthology.orchestra.run.smithi171.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB]
2024-05-17T14:52:48.096 INFO:teuthology.orchestra.run.smithi171.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB]
2024-05-17T14:52:48.098 INFO:teuthology.orchestra.run.smithi171.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB]
2024-05-17T14:52:48.098 INFO:teuthology.orchestra.run.smithi171.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB]
2024-05-17T14:52:48.125 INFO:teuthology.orchestra.run.smithi171.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B]
2024-05-17T14:52:48.125 INFO:teuthology.orchestra.run.smithi171.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB]
2024-05-17T14:52:48.157 INFO:teuthology.orchestra.run.smithi171.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB]
2024-05-17T14:52:48.162 INFO:teuthology.orchestra.run.smithi171.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB]
2024-05-17T14:52:48.163 INFO:teuthology.orchestra.run.smithi171.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB]
2024-05-17T14:52:48.164 INFO:teuthology.orchestra.run.smithi171.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB]
2024-05-17T14:52:48.164 INFO:teuthology.orchestra.run.smithi171.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB]
2024-05-17T14:52:48.166 INFO:teuthology.orchestra.run.smithi171.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB]
2024-05-17T14:52:48.173 INFO:teuthology.orchestra.run.smithi171.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB]
2024-05-17T14:52:48.196 INFO:teuthology.orchestra.run.smithi171.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB]
2024-05-17T14:52:48.199 INFO:teuthology.orchestra.run.smithi171.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB]
2024-05-17T14:52:48.200 INFO:teuthology.orchestra.run.smithi171.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB]
2024-05-17T14:52:48.222 INFO:teuthology.orchestra.run.smithi171.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB]
2024-05-17T14:52:48.228 INFO:teuthology.orchestra.run.smithi171.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB]
2024-05-17T14:52:48.230 INFO:teuthology.orchestra.run.smithi171.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB]
2024-05-17T14:52:48.231 INFO:teuthology.orchestra.run.smithi171.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB]
2024-05-17T14:52:48.242 INFO:teuthology.orchestra.run.smithi171.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB]
2024-05-17T14:52:48.243 INFO:teuthology.orchestra.run.smithi171.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB]
2024-05-17T14:52:48.250 INFO:teuthology.orchestra.run.smithi171.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B]
2024-05-17T14:52:48.252 INFO:teuthology.orchestra.run.smithi171.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB]
2024-05-17T14:52:48.253 INFO:teuthology.orchestra.run.smithi171.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB]
2024-05-17T14:52:48.254 INFO:teuthology.orchestra.run.smithi171.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB]
2024-05-17T14:52:48.279 INFO:teuthology.orchestra.run.smithi072.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB]
2024-05-17T14:52:48.290 INFO:teuthology.orchestra.run.smithi171.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB]
2024-05-17T14:52:48.292 INFO:teuthology.orchestra.run.smithi171.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB]
2024-05-17T14:52:48.296 INFO:teuthology.orchestra.run.smithi072.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB]
2024-05-17T14:52:48.308 INFO:teuthology.orchestra.run.smithi171.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB]
2024-05-17T14:52:48.309 INFO:teuthology.orchestra.run.smithi171.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB]
2024-05-17T14:52:48.309 INFO:teuthology.orchestra.run.smithi171.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB]
2024-05-17T14:52:48.314 INFO:teuthology.orchestra.run.smithi171.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB]
2024-05-17T14:52:48.315 INFO:teuthology.orchestra.run.smithi171.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB]
2024-05-17T14:52:48.326 INFO:teuthology.orchestra.run.smithi072.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB]
2024-05-17T14:52:48.335 INFO:teuthology.orchestra.run.smithi072.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB]
2024-05-17T14:52:48.336 INFO:teuthology.orchestra.run.smithi072.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB]
2024-05-17T14:52:48.337 INFO:teuthology.orchestra.run.smithi072.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB]
2024-05-17T14:52:48.355 INFO:teuthology.orchestra.run.smithi072.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB]
2024-05-17T14:52:48.523 INFO:teuthology.orchestra.run.smithi072.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB]
2024-05-17T14:52:48.861 INFO:teuthology.orchestra.run.smithi171.stdout:Get:89 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.0.0-3735-gee59d86a-1jammy [3,225 kB]
2024-05-17T14:52:48.925 INFO:teuthology.orchestra.run.smithi072.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB]
2024-05-17T14:52:48.959 INFO:teuthology.orchestra.run.smithi072.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5,034 B]
2024-05-17T14:52:48.961 INFO:teuthology.orchestra.run.smithi072.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB]
2024-05-17T14:52:49.041 INFO:teuthology.orchestra.run.smithi072.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB]
2024-05-17T14:52:49.092 INFO:teuthology.orchestra.run.smithi072.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB]
2024-05-17T14:52:49.114 INFO:teuthology.orchestra.run.smithi072.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB]
2024-05-17T14:52:49.120 INFO:teuthology.orchestra.run.smithi072.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B]
2024-05-17T14:52:49.121 INFO:teuthology.orchestra.run.smithi072.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB]
2024-05-17T14:52:49.132 INFO:teuthology.orchestra.run.smithi072.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B]
2024-05-17T14:52:49.133 INFO:teuthology.orchestra.run.smithi072.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B]
2024-05-17T14:52:49.299 INFO:teuthology.orchestra.run.smithi072.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB]
2024-05-17T14:52:49.382 INFO:teuthology.orchestra.run.smithi072.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB]
2024-05-17T14:52:49.410 INFO:teuthology.orchestra.run.smithi072.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B]
2024-05-17T14:52:49.428 INFO:teuthology.orchestra.run.smithi072.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B]
2024-05-17T14:52:49.472 INFO:teuthology.orchestra.run.smithi072.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB]
2024-05-17T14:52:49.682 INFO:teuthology.orchestra.run.smithi072.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB]
2024-05-17T14:52:49.697 INFO:teuthology.orchestra.run.smithi072.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB]
2024-05-17T14:52:49.704 INFO:teuthology.orchestra.run.smithi072.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB]
2024-05-17T14:52:49.709 INFO:teuthology.orchestra.run.smithi072.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB]
2024-05-17T14:52:49.731 INFO:teuthology.orchestra.run.smithi072.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB]
2024-05-17T14:52:49.734 INFO:teuthology.orchestra.run.smithi072.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B]
2024-05-17T14:52:49.780 INFO:teuthology.orchestra.run.smithi171.stdout:Get:90 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.0.0-3735-gee59d86a-1jammy [713 kB]
2024-05-17T14:52:49.900 INFO:teuthology.orchestra.run.smithi072.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB]
2024-05-17T14:52:49.996 INFO:teuthology.orchestra.run.smithi171.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.0.0-3735-gee59d86a-1jammy [357 kB]
2024-05-17T14:52:50.076 INFO:teuthology.orchestra.run.smithi171.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.0.0-3735-gee59d86a-1jammy [32.8 kB]
2024-05-17T14:52:50.078 INFO:teuthology.orchestra.run.smithi171.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.0.0-3735-gee59d86a-1jammy [180 kB]
2024-05-17T14:52:50.149 INFO:teuthology.orchestra.run.smithi171.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.0.0-3735-gee59d86a-1jammy [67.4 kB]
2024-05-17T14:52:50.152 INFO:teuthology.orchestra.run.smithi171.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.0.0-3735-gee59d86a-1jammy [334 kB]
2024-05-17T14:52:50.231 INFO:teuthology.orchestra.run.smithi171.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.0.0-3735-gee59d86a-1jammy [7,881 kB]
2024-05-17T14:52:50.232 INFO:teuthology.orchestra.run.smithi072.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB]
2024-05-17T14:52:50.263 INFO:teuthology.orchestra.run.smithi072.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB]
2024-05-17T14:52:50.274 INFO:teuthology.orchestra.run.smithi072.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB]
2024-05-17T14:52:50.476 INFO:teuthology.orchestra.run.smithi072.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B]
2024-05-17T14:52:50.477 INFO:teuthology.orchestra.run.smithi072.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB]
2024-05-17T14:52:50.483 INFO:teuthology.orchestra.run.smithi072.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB]
2024-05-17T14:52:50.489 INFO:teuthology.orchestra.run.smithi072.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB]
2024-05-17T14:52:50.504 INFO:teuthology.orchestra.run.smithi072.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB]
2024-05-17T14:52:50.526 INFO:teuthology.orchestra.run.smithi072.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB]
2024-05-17T14:52:50.532 INFO:teuthology.orchestra.run.smithi072.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB]
2024-05-17T14:52:50.707 INFO:teuthology.orchestra.run.smithi072.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB]
2024-05-17T14:52:50.969 INFO:teuthology.orchestra.run.smithi072.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB]
2024-05-17T14:52:51.148 INFO:teuthology.orchestra.run.smithi072.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB]
2024-05-17T14:52:51.163 INFO:teuthology.orchestra.run.smithi072.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB]
2024-05-17T14:52:51.182 INFO:teuthology.orchestra.run.smithi072.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB]
2024-05-17T14:52:51.247 INFO:teuthology.orchestra.run.smithi072.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB]
2024-05-17T14:52:51.477 INFO:teuthology.orchestra.run.smithi072.stdout:Get:52 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.0.0-3735-gee59d86a-1jammy [3,225 kB]
2024-05-17T14:52:51.534 INFO:teuthology.orchestra.run.smithi072.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB]
2024-05-17T14:52:51.537 INFO:teuthology.orchestra.run.smithi072.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB]
2024-05-17T14:52:51.539 INFO:teuthology.orchestra.run.smithi072.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB]
2024-05-17T14:52:51.539 INFO:teuthology.orchestra.run.smithi072.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB]
2024-05-17T14:52:51.542 INFO:teuthology.orchestra.run.smithi072.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB]
2024-05-17T14:52:51.762 INFO:teuthology.orchestra.run.smithi072.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB]
2024-05-17T14:52:52.199 INFO:teuthology.orchestra.run.smithi072.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB]
2024-05-17T14:52:52.207 INFO:teuthology.orchestra.run.smithi072.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB]
2024-05-17T14:52:52.495 INFO:teuthology.orchestra.run.smithi072.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B]
2024-05-17T14:52:52.496 INFO:teuthology.orchestra.run.smithi072.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB]
2024-05-17T14:52:52.498 INFO:teuthology.orchestra.run.smithi072.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB]
2024-05-17T14:52:52.499 INFO:teuthology.orchestra.run.smithi072.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB]
2024-05-17T14:52:52.501 INFO:teuthology.orchestra.run.smithi072.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB]
2024-05-17T14:52:52.503 INFO:teuthology.orchestra.run.smithi072.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB]
2024-05-17T14:52:52.504 INFO:teuthology.orchestra.run.smithi072.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB]
2024-05-17T14:52:52.506 INFO:teuthology.orchestra.run.smithi072.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB]
2024-05-17T14:52:52.604 INFO:teuthology.orchestra.run.smithi171.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.0.0-3735-gee59d86a-1jammy [112 kB]
2024-05-17T14:52:52.606 INFO:teuthology.orchestra.run.smithi171.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.0.0-3735-gee59d86a-1jammy [467 kB]
2024-05-17T14:52:52.688 INFO:teuthology.orchestra.run.smithi072.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB]
2024-05-17T14:52:52.761 INFO:teuthology.orchestra.run.smithi171.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.0.0-3735-gee59d86a-1jammy [27.8 MB]
2024-05-17T14:52:53.093 INFO:teuthology.orchestra.run.smithi072.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB]
2024-05-17T14:52:53.140 INFO:teuthology.orchestra.run.smithi072.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB]
2024-05-17T14:52:53.174 INFO:teuthology.orchestra.run.smithi072.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB]
2024-05-17T14:52:53.271 INFO:teuthology.orchestra.run.smithi072.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB]
2024-05-17T14:52:53.311 INFO:teuthology.orchestra.run.smithi072.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB]
2024-05-17T14:52:53.323 INFO:teuthology.orchestra.run.smithi072.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB]
2024-05-17T14:52:53.330 INFO:teuthology.orchestra.run.smithi072.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB]
2024-05-17T14:52:53.375 INFO:teuthology.orchestra.run.smithi072.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB]
2024-05-17T14:52:53.379 INFO:teuthology.orchestra.run.smithi072.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB]
2024-05-17T14:52:53.400 INFO:teuthology.orchestra.run.smithi072.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B]
2024-05-17T14:52:53.565 INFO:teuthology.orchestra.run.smithi072.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB]
2024-05-17T14:52:53.839 INFO:teuthology.orchestra.run.smithi072.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB]
2024-05-17T14:52:53.855 INFO:teuthology.orchestra.run.smithi072.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB]
2024-05-17T14:52:54.010 INFO:teuthology.orchestra.run.smithi072.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB]
2024-05-17T14:52:54.039 INFO:teuthology.orchestra.run.smithi072.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB]
2024-05-17T14:52:54.137 INFO:teuthology.orchestra.run.smithi072.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.0.0-3735-gee59d86a-1jammy [713 kB]
2024-05-17T14:52:54.197 INFO:teuthology.orchestra.run.smithi072.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB]
2024-05-17T14:52:54.200 INFO:teuthology.orchestra.run.smithi072.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB]
2024-05-17T14:52:54.205 INFO:teuthology.orchestra.run.smithi072.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB]
2024-05-17T14:52:54.225 INFO:teuthology.orchestra.run.smithi072.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB]
2024-05-17T14:52:54.229 INFO:teuthology.orchestra.run.smithi072.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB]
2024-05-17T14:52:54.640 INFO:teuthology.orchestra.run.smithi072.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.0.0-3735-gee59d86a-1jammy [357 kB]
2024-05-17T14:52:54.892 INFO:teuthology.orchestra.run.smithi072.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.0.0-3735-gee59d86a-1jammy [32.8 kB]
2024-05-17T14:52:54.906 INFO:teuthology.orchestra.run.smithi072.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.0.0-3735-gee59d86a-1jammy [180 kB]
2024-05-17T14:52:55.024 INFO:teuthology.orchestra.run.smithi072.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.0.0-3735-gee59d86a-1jammy [67.4 kB]
2024-05-17T14:52:55.078 INFO:teuthology.orchestra.run.smithi072.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.0.0-3735-gee59d86a-1jammy [334 kB]
2024-05-17T14:52:55.311 INFO:teuthology.orchestra.run.smithi072.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.0.0-3735-gee59d86a-1jammy [7,881 kB]
2024-05-17T14:52:58.515 INFO:teuthology.orchestra.run.smithi072.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.0.0-3735-gee59d86a-1jammy [112 kB]
2024-05-17T14:52:58.538 INFO:teuthology.orchestra.run.smithi072.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.0.0-3735-gee59d86a-1jammy [467 kB]
2024-05-17T14:52:58.596 INFO:teuthology.orchestra.run.smithi072.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.0.0-3735-gee59d86a-1jammy [27.8 MB]
2024-05-17T14:52:59.187 INFO:teuthology.orchestra.run.smithi171.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.0.0-3735-gee59d86a-1jammy [5,093 kB]
2024-05-17T14:52:59.518 INFO:teuthology.orchestra.run.smithi171.stdout:Get:101 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.0.0-3735-gee59d86a-1jammy [245 kB]
2024-05-17T14:52:59.537 INFO:teuthology.orchestra.run.smithi171.stdout:Get:102 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.0.0-3735-gee59d86a-1jammy [124 kB]
2024-05-17T14:52:59.541 INFO:teuthology.orchestra.run.smithi171.stdout:Get:103 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.0.0-3735-gee59d86a-1jammy [1,545 kB]
2024-05-17T14:52:59.661 INFO:teuthology.orchestra.run.smithi171.stdout:Get:104 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.0.0-3735-gee59d86a-1jammy [6,260 kB]
2024-05-17T14:53:00.117 INFO:teuthology.orchestra.run.smithi171.stdout:Get:105 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.0.0-3735-gee59d86a-1jammy [22.8 MB]
2024-05-17T14:53:01.729 INFO:teuthology.orchestra.run.smithi171.stdout:Get:106 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.0.0-3735-gee59d86a-1jammy [14.0 kB]
2024-05-17T14:53:01.730 INFO:teuthology.orchestra.run.smithi171.stdout:Get:107 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.0.0-3735-gee59d86a-1jammy [870 kB]
2024-05-17T14:53:01.753 INFO:teuthology.orchestra.run.smithi171.stdout:Get:108 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.0.0-3735-gee59d86a-1jammy [2,441 kB]
2024-05-17T14:53:01.897 INFO:teuthology.orchestra.run.smithi171.stdout:Get:109 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.0.0-3735-gee59d86a-1jammy [793 kB]
2024-05-17T14:53:01.941 INFO:teuthology.orchestra.run.smithi171.stdout:Get:110 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.0.0-3735-gee59d86a-1jammy [149 kB]
2024-05-17T14:53:01.956 INFO:teuthology.orchestra.run.smithi171.stdout:Get:111 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.0.0-3735-gee59d86a-1jammy [2,222 kB]
2024-05-17T14:53:02.085 INFO:teuthology.orchestra.run.smithi171.stdout:Get:112 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.0.0-3735-gee59d86a-1jammy [8,625 kB]
2024-05-17T14:53:02.174 INFO:teuthology.orchestra.run.smithi072.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.0.0-3735-gee59d86a-1jammy [5,093 kB]
2024-05-17T14:53:02.373 INFO:teuthology.orchestra.run.smithi072.stdout:Get:101 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.0.0-3735-gee59d86a-1jammy [245 kB]
2024-05-17T14:53:02.385 INFO:teuthology.orchestra.run.smithi072.stdout:Get:102 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.0.0-3735-gee59d86a-1jammy [124 kB]
2024-05-17T14:53:02.394 INFO:teuthology.orchestra.run.smithi072.stdout:Get:103 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.0.0-3735-gee59d86a-1jammy [1,545 kB]
2024-05-17T14:53:02.486 INFO:teuthology.orchestra.run.smithi072.stdout:Get:104 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.0.0-3735-gee59d86a-1jammy [6,260 kB]
2024-05-17T14:53:02.680 INFO:teuthology.orchestra.run.smithi171.stdout:Get:113 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.0.0-3735-gee59d86a-1jammy [14.2 kB]
2024-05-17T14:53:02.682 INFO:teuthology.orchestra.run.smithi171.stdout:Get:114 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.0.0-3735-gee59d86a-1jammy [54.6 MB]
2024-05-17T14:53:02.848 INFO:teuthology.orchestra.run.smithi072.stdout:Get:105 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.0.0-3735-gee59d86a-1jammy [22.8 MB]
2024-05-17T14:53:04.174 INFO:teuthology.orchestra.run.smithi072.stdout:Get:106 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.0.0-3735-gee59d86a-1jammy [14.0 kB]
2024-05-17T14:53:04.175 INFO:teuthology.orchestra.run.smithi072.stdout:Get:107 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.0.0-3735-gee59d86a-1jammy [870 kB]
2024-05-17T14:53:04.194 INFO:teuthology.orchestra.run.smithi072.stdout:Get:108 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.0.0-3735-gee59d86a-1jammy [2,441 kB]
2024-05-17T14:53:04.263 INFO:teuthology.orchestra.run.smithi072.stdout:Get:109 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.0.0-3735-gee59d86a-1jammy [793 kB]
2024-05-17T14:53:04.302 INFO:teuthology.orchestra.run.smithi072.stdout:Get:110 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.0.0-3735-gee59d86a-1jammy [149 kB]
2024-05-17T14:53:04.306 INFO:teuthology.orchestra.run.smithi072.stdout:Get:111 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.0.0-3735-gee59d86a-1jammy [2,222 kB]
2024-05-17T14:53:04.417 INFO:teuthology.orchestra.run.smithi072.stdout:Get:112 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.0.0-3735-gee59d86a-1jammy [8,625 kB]
2024-05-17T14:53:04.874 INFO:teuthology.orchestra.run.smithi072.stdout:Get:113 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.0.0-3735-gee59d86a-1jammy [14.2 kB]
2024-05-17T14:53:04.885 INFO:teuthology.orchestra.run.smithi072.stdout:Get:114 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.0.0-3735-gee59d86a-1jammy [54.6 MB]
2024-05-17T14:53:08.039 INFO:teuthology.orchestra.run.smithi171.stdout:Get:115 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.0.0-3735-gee59d86a-1jammy [124 kB]
2024-05-17T14:53:08.041 INFO:teuthology.orchestra.run.smithi171.stdout:Get:116 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.0.0-3735-gee59d86a-1jammy [38.9 kB]
2024-05-17T14:53:08.042 INFO:teuthology.orchestra.run.smithi171.stdout:Get:117 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.0.0-3735-gee59d86a-1jammy [15.7 MB]
2024-05-17T14:53:08.120 INFO:teuthology.orchestra.run.smithi072.stdout:Get:115 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.0.0-3735-gee59d86a-1jammy [124 kB]
2024-05-17T14:53:08.123 INFO:teuthology.orchestra.run.smithi072.stdout:Get:116 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.0.0-3735-gee59d86a-1jammy [38.9 kB]
2024-05-17T14:53:08.124 INFO:teuthology.orchestra.run.smithi072.stdout:Get:117 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.0.0-3735-gee59d86a-1jammy [15.7 MB]
2024-05-17T14:53:08.822 INFO:teuthology.orchestra.run.smithi072.stdout:Get:118 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.0.0-3735-gee59d86a-1jammy [91.2 kB]
2024-05-17T14:53:08.824 INFO:teuthology.orchestra.run.smithi072.stdout:Get:119 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.0.0-3735-gee59d86a-1jammy [3,124 kB]
2024-05-17T14:53:09.068 INFO:teuthology.orchestra.run.smithi171.stdout:Get:118 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.0.0-3735-gee59d86a-1jammy [91.2 kB]
2024-05-17T14:53:09.070 INFO:teuthology.orchestra.run.smithi171.stdout:Get:119 https://1.chacra.ceph.com/r/ceph/wip-dis-testing/ee59d86aad04089481e55c1a15aa5871339935b4/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.0.0-3735-gee59d86a-1jammy [3,124 kB]
2024-05-17T14:53:09.448 INFO:teuthology.orchestra.run.smithi072.stdout:Fetched 192 MB in 22s (8,922 kB/s)
2024-05-17T14:53:09.612 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package liblttng-ust1:amd64.
2024-05-17T14:53:09.711 INFO:teuthology.orchestra.run.smithi171.stdout:Fetched 192 MB in 22s (8,776 kB/s)
2024-05-17T14:53:09.712 INFO:teuthology.orchestra.run.smithi072.stdout:(Reading database ... 
(Reading database ... 5%
(Reading database ... 10%
(Reading database ... 15%
(Reading database ... 20%
(Reading database ... 25%
(Reading database ... 30%
(Reading database ... 35%
(Reading database ... 40%
(Reading database ... 45%
(Reading database ... 50%
(Reading database ... 55%
(Reading database ... 60%
(Reading database ... 65%
(Reading database ... 70%
(Reading database ... 75%
(Reading database ... 80%
(Reading database ... 85%
(Reading database ... 90%
(Reading database ... 95%
(Reading database ... 100%
(Reading database ... 165990 files and directories currently installed.)
2024-05-17T14:53:09.718 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ...
2024-05-17T14:53:09.856 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package liblttng-ust1:amd64.
2024-05-17T14:53:09.886 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ...
2024-05-17T14:53:09.953 INFO:teuthology.orchestra.run.smithi171.stdout:(Reading database ... 
(Reading database ... 5%
(Reading database ... 10%
(Reading database ... 15%
(Reading database ... 20%
(Reading database ... 25%
(Reading database ... 30%
(Reading database ... 35%
(Reading database ... 40%
(Reading database ... 45%
(Reading database ... 50%
(Reading database ... 55%
(Reading database ... 60%
(Reading database ... 65%
(Reading database ... 70%
(Reading database ... 75%
(Reading database ... 80%
(Reading database ... 85%
(Reading database ... 90%
(Reading database ... 95%
(Reading database ... 100%
(Reading database ... 165990 files and directories currently installed.)
2024-05-17T14:53:09.958 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ...
2024-05-17T14:53:10.038 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ...
2024-05-17T14:53:10.210 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libdouble-conversion3:amd64.
2024-05-17T14:53:10.236 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ...
2024-05-17T14:53:10.277 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ...
2024-05-17T14:53:10.366 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libdouble-conversion3:amd64.
2024-05-17T14:53:10.393 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ...
2024-05-17T14:53:10.433 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ...
2024-05-17T14:53:10.607 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libpcre2-16-0:amd64.
2024-05-17T14:53:10.634 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ...
2024-05-17T14:53:10.674 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ...
2024-05-17T14:53:10.746 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libpcre2-16-0:amd64.
2024-05-17T14:53:10.773 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ...
2024-05-17T14:53:10.813 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ...
2024-05-17T14:53:11.123 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libqt5core5a:amd64.
2024-05-17T14:53:11.150 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ...
2024-05-17T14:53:11.198 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:53:11.208 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libqt5core5a:amd64.
2024-05-17T14:53:11.235 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ...
2024-05-17T14:53:11.283 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:53:11.631 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libqt5dbus5:amd64.
2024-05-17T14:53:11.657 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ...
2024-05-17T14:53:11.677 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libqt5dbus5:amd64.
2024-05-17T14:53:11.698 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:53:11.704 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ...
2024-05-17T14:53:11.744 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:53:12.028 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libqt5network5:amd64.
2024-05-17T14:53:12.055 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ...
2024-05-17T14:53:12.089 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libqt5network5:amd64.
2024-05-17T14:53:12.096 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:53:12.116 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ...
2024-05-17T14:53:12.155 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:53:12.433 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libthrift-0.16.0:amd64.
2024-05-17T14:53:12.449 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ...
2024-05-17T14:53:12.460 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libthrift-0.16.0:amd64.
2024-05-17T14:53:12.487 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ...
2024-05-17T14:53:12.491 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ...
2024-05-17T14:53:12.527 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ...
2024-05-17T14:53:12.828 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package librados2.
2024-05-17T14:53:12.854 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../007-librados2_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:12.858 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package librados2.
2024-05-17T14:53:12.885 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../007-librados2_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:12.894 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking librados2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:12.925 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking librados2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:13.305 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package librbd1.
2024-05-17T14:53:13.329 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../008-librbd1_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:13.364 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking librbd1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:13.439 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package librbd1.
2024-05-17T14:53:13.454 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../008-librbd1_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:13.489 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking librbd1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:13.800 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libcephfs2.
2024-05-17T14:53:13.827 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../009-libcephfs2_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:13.866 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libcephfs2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:13.971 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libcephfs2.
2024-05-17T14:53:14.146 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../009-libcephfs2_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:14.146 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libcephfs2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:14.152 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-rados.
2024-05-17T14:53:14.179 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../010-python3-rados_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:14.222 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-rados (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:14.326 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-rados.
2024-05-17T14:53:14.353 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../010-python3-rados_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:14.394 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-rados (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:14.522 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-ceph-argparse.
2024-05-17T14:53:14.549 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../011-python3-ceph-argparse_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:14.588 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-ceph-argparse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:14.723 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-ceph-argparse.
2024-05-17T14:53:14.750 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../011-python3-ceph-argparse_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:14.799 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-cephfs.
2024-05-17T14:53:14.815 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../012-python3-cephfs_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:14.849 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-cephfs (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:14.968 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-ceph-argparse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:15.160 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-ceph-common.
2024-05-17T14:53:15.187 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../013-python3-ceph-common_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:15.221 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-cephfs.
2024-05-17T14:53:15.227 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:15.248 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../012-python3-cephfs_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:15.288 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-cephfs (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:15.580 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-wcwidth.
2024-05-17T14:53:15.607 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../014-python3-wcwidth_0.2.5+dfsg1-1_all.deb ...
2024-05-17T14:53:15.610 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-ceph-common.
2024-05-17T14:53:15.636 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../013-python3-ceph-common_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:15.647 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ...
2024-05-17T14:53:15.676 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:15.966 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-prettytable.
2024-05-17T14:53:15.993 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../015-python3-prettytable_2.5.0-2_all.deb ...
2024-05-17T14:53:16.000 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-wcwidth.
2024-05-17T14:53:16.026 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../014-python3-wcwidth_0.2.5+dfsg1-1_all.deb ...
2024-05-17T14:53:16.032 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-prettytable (2.5.0-2) ...
2024-05-17T14:53:16.067 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ...
2024-05-17T14:53:16.269 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-rbd.
2024-05-17T14:53:16.284 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../016-python3-rbd_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:16.318 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-rbd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:16.365 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-prettytable.
2024-05-17T14:53:16.381 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../015-python3-prettytable_2.5.0-2_all.deb ...
2024-05-17T14:53:16.415 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-prettytable (2.5.0-2) ...
2024-05-17T14:53:16.663 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package librdkafka1:amd64.
2024-05-17T14:53:16.670 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-rbd.
2024-05-17T14:53:16.690 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../017-librdkafka1_1.8.0-1build1_amd64.deb ...
2024-05-17T14:53:16.696 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../016-python3-rbd_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:16.730 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ...
2024-05-17T14:53:16.736 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-rbd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:17.016 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libreadline-dev:amd64.
2024-05-17T14:53:17.043 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../018-libreadline-dev_8.1.2-1_amd64.deb ...
2024-05-17T14:53:17.060 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package librdkafka1:amd64.
2024-05-17T14:53:17.082 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ...
2024-05-17T14:53:17.087 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../017-librdkafka1_1.8.0-1build1_amd64.deb ...
2024-05-17T14:53:17.127 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ...
2024-05-17T14:53:17.385 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package liblua5.3-dev:amd64.
2024-05-17T14:53:17.412 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../019-liblua5.3-dev_5.3.6-1build1_amd64.deb ...
2024-05-17T14:53:17.414 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libreadline-dev:amd64.
2024-05-17T14:53:17.441 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../018-libreadline-dev_8.1.2-1_amd64.deb ...
2024-05-17T14:53:17.451 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ...
2024-05-17T14:53:17.481 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ...
2024-05-17T14:53:17.747 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package lua5.1.
2024-05-17T14:53:17.774 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../020-lua5.1_5.1.5-8.1build4_amd64.deb ...
2024-05-17T14:53:17.813 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package liblua5.3-dev:amd64.
2024-05-17T14:53:17.814 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ...
2024-05-17T14:53:17.828 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../019-liblua5.3-dev_5.3.6-1build1_amd64.deb ...
2024-05-17T14:53:17.905 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ...
2024-05-17T14:53:18.125 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package lua-any.
2024-05-17T14:53:18.140 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../021-lua-any_27ubuntu1_all.deb ...
2024-05-17T14:53:18.183 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking lua-any (27ubuntu1) ...
2024-05-17T14:53:18.238 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package lua5.1.
2024-05-17T14:53:18.264 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../020-lua5.1_5.1.5-8.1build4_amd64.deb ...
2024-05-17T14:53:18.306 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ...
2024-05-17T14:53:18.427 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package zip.
2024-05-17T14:53:18.443 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../022-zip_3.0-12build2_amd64.deb ...
2024-05-17T14:53:18.485 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking zip (3.0-12build2) ...
2024-05-17T14:53:18.609 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package lua-any.
2024-05-17T14:53:18.625 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../021-lua-any_27ubuntu1_all.deb ...
2024-05-17T14:53:18.661 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking lua-any (27ubuntu1) ...
2024-05-17T14:53:18.755 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package unzip.
2024-05-17T14:53:18.771 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../023-unzip_6.0-26ubuntu3.2_amd64.deb ...
2024-05-17T14:53:18.805 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking unzip (6.0-26ubuntu3.2) ...
2024-05-17T14:53:18.908 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package zip.
2024-05-17T14:53:18.934 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../022-zip_3.0-12build2_amd64.deb ...
2024-05-17T14:53:18.975 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking zip (3.0-12build2) ...
2024-05-17T14:53:19.184 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package luarocks.
2024-05-17T14:53:19.211 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../024-luarocks_3.8.0+dfsg1-1_all.deb ...
2024-05-17T14:53:19.251 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ...
2024-05-17T14:53:19.290 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package unzip.
2024-05-17T14:53:19.316 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../023-unzip_6.0-26ubuntu3.2_amd64.deb ...
2024-05-17T14:53:19.357 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking unzip (6.0-26ubuntu3.2) ...
2024-05-17T14:53:19.707 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package luarocks.
2024-05-17T14:53:19.734 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../024-luarocks_3.8.0+dfsg1-1_all.deb ...
2024-05-17T14:53:19.774 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ...
2024-05-17T14:53:20.356 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package librgw2.
2024-05-17T14:53:20.383 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../025-librgw2_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:20.459 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package librgw2.
2024-05-17T14:53:20.486 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../025-librgw2_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:20.525 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking librgw2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:20.572 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking librgw2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:21.153 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-rgw.
2024-05-17T14:53:21.168 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../026-python3-rgw_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:21.203 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-rgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:21.514 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package liboath0:amd64.
2024-05-17T14:53:21.541 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../027-liboath0_2.6.7-3build1_amd64.deb ...
2024-05-17T14:53:21.581 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ...
2024-05-17T14:53:21.875 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libradosstriper1.
2024-05-17T14:53:21.902 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../028-libradosstriper1_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:21.942 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libradosstriper1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:22.370 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-common.
2024-05-17T14:53:22.397 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../029-ceph-common_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:22.437 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:22.504 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-rgw.
2024-05-17T14:53:22.531 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../026-python3-rgw_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:22.571 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-rgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:22.944 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package liboath0:amd64.
2024-05-17T14:53:22.971 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../027-liboath0_2.6.7-3build1_amd64.deb ...
2024-05-17T14:53:23.011 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ...
2024-05-17T14:53:23.316 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libradosstriper1.
2024-05-17T14:53:23.343 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../028-libradosstriper1_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:23.383 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libradosstriper1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:23.824 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-common.
2024-05-17T14:53:23.850 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../029-ceph-common_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:23.890 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:24.008 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-base.
2024-05-17T14:53:24.035 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../030-ceph-base_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:24.076 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-base (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:24.771 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-jaraco.functools.
2024-05-17T14:53:24.798 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../031-python3-jaraco.functools_3.4.0-2_all.deb ...
2024-05-17T14:53:24.838 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ...
2024-05-17T14:53:25.149 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-cheroot.
2024-05-17T14:53:25.176 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../032-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ...
2024-05-17T14:53:25.216 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ...
2024-05-17T14:53:25.544 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-jaraco.classes.
2024-05-17T14:53:25.570 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../033-python3-jaraco.classes_3.2.1-3_all.deb ...
2024-05-17T14:53:25.610 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ...
2024-05-17T14:53:25.877 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-base.
2024-05-17T14:53:25.903 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../030-ceph-base_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:25.921 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-jaraco.text.
2024-05-17T14:53:25.948 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../034-python3-jaraco.text_3.6.0-2_all.deb ...
2024-05-17T14:53:25.953 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-base (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:25.988 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-jaraco.text (3.6.0-2) ...
2024-05-17T14:53:26.282 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-jaraco.collections.
2024-05-17T14:53:26.309 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../035-python3-jaraco.collections_3.4.0-2_all.deb ...
2024-05-17T14:53:26.341 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ...
2024-05-17T14:53:26.611 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-jaraco.functools.
2024-05-17T14:53:26.627 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-tempora.
2024-05-17T14:53:26.638 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../031-python3-jaraco.functools_3.4.0-2_all.deb ...
2024-05-17T14:53:26.654 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../036-python3-tempora_4.1.2-1_all.deb ...
2024-05-17T14:53:26.678 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ...
2024-05-17T14:53:26.693 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-tempora (4.1.2-1) ...
2024-05-17T14:53:26.951 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-cheroot.
2024-05-17T14:53:26.978 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../032-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ...
2024-05-17T14:53:27.005 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-portend.
2024-05-17T14:53:27.018 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ...
2024-05-17T14:53:27.031 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../037-python3-portend_3.0.0-1_all.deb ...
2024-05-17T14:53:27.071 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-portend (3.0.0-1) ...
2024-05-17T14:53:27.332 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-zc.lockfile.
2024-05-17T14:53:27.348 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-jaraco.classes.
2024-05-17T14:53:27.359 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../038-python3-zc.lockfile_2.0-1_all.deb ...
2024-05-17T14:53:27.375 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../033-python3-jaraco.classes_3.2.1-3_all.deb ...
2024-05-17T14:53:27.399 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-zc.lockfile (2.0-1) ...
2024-05-17T14:53:27.415 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ...
2024-05-17T14:53:27.652 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-cherrypy3.
2024-05-17T14:53:27.679 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../039-python3-cherrypy3_18.6.1-4_all.deb ...
2024-05-17T14:53:27.718 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ...
2024-05-17T14:53:27.730 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-jaraco.text.
2024-05-17T14:53:27.745 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../034-python3-jaraco.text_3.6.0-2_all.deb ...
2024-05-17T14:53:27.780 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-jaraco.text (3.6.0-2) ...
2024-05-17T14:53:28.047 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-natsort.
2024-05-17T14:53:28.074 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../040-python3-natsort_8.0.2-1_all.deb ...
2024-05-17T14:53:28.078 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-jaraco.collections.
2024-05-17T14:53:28.105 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../035-python3-jaraco.collections_3.4.0-2_all.deb ...
2024-05-17T14:53:28.113 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-natsort (8.0.2-1) ...
2024-05-17T14:53:28.145 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ...
2024-05-17T14:53:28.383 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-logutils.
2024-05-17T14:53:28.410 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../041-python3-logutils_0.3.3-8_all.deb ...
2024-05-17T14:53:28.442 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-tempora.
2024-05-17T14:53:28.458 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-logutils (0.3.3-8) ...
2024-05-17T14:53:28.469 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../036-python3-tempora_4.1.2-1_all.deb ...
2024-05-17T14:53:28.509 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-tempora (4.1.2-1) ...
2024-05-17T14:53:28.745 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-markupsafe.
2024-05-17T14:53:28.771 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../042-python3-markupsafe_2.0.1-2build1_amd64.deb ...
2024-05-17T14:53:28.807 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-portend.
2024-05-17T14:53:28.811 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ...
2024-05-17T14:53:28.834 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../037-python3-portend_3.0.0-1_all.deb ...
2024-05-17T14:53:28.882 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-portend (3.0.0-1) ...
2024-05-17T14:53:29.139 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-mako.
2024-05-17T14:53:29.166 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ...
2024-05-17T14:53:29.179 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-zc.lockfile.
2024-05-17T14:53:29.205 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ...
2024-05-17T14:53:29.206 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../038-python3-zc.lockfile_2.0-1_all.deb ...
2024-05-17T14:53:29.247 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-zc.lockfile (2.0-1) ...
2024-05-17T14:53:29.517 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-simplegeneric.
2024-05-17T14:53:29.527 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-cherrypy3.
2024-05-17T14:53:29.544 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ...
2024-05-17T14:53:29.554 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../039-python3-cherrypy3_18.6.1-4_all.deb ...
2024-05-17T14:53:29.583 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-simplegeneric (0.8.1-3) ...
2024-05-17T14:53:29.594 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ...
2024-05-17T14:53:29.853 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-singledispatch.
2024-05-17T14:53:29.880 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ...
2024-05-17T14:53:29.911 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ...
2024-05-17T14:53:29.927 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-natsort.
2024-05-17T14:53:29.954 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../040-python3-natsort_8.0.2-1_all.deb ...
2024-05-17T14:53:30.002 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-natsort (8.0.2-1) ...
2024-05-17T14:53:30.190 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-webob.
2024-05-17T14:53:30.216 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1_all.deb ...
2024-05-17T14:53:30.255 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-webob (1:1.8.6-1.1) ...
2024-05-17T14:53:30.317 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-logutils.
2024-05-17T14:53:30.344 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../041-python3-logutils_0.3.3-8_all.deb ...
2024-05-17T14:53:30.387 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-logutils (0.3.3-8) ...
2024-05-17T14:53:30.609 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-waitress.
2024-05-17T14:53:30.636 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1_all.deb ...
2024-05-17T14:53:30.678 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ...
2024-05-17T14:53:30.914 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-markupsafe.
2024-05-17T14:53:30.941 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../042-python3-markupsafe_2.0.1-2build1_amd64.deb ...
2024-05-17T14:53:30.981 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ...
2024-05-17T14:53:30.986 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-tempita.
2024-05-17T14:53:31.013 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ...
2024-05-17T14:53:31.053 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ...
2024-05-17T14:53:31.263 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-mako.
2024-05-17T14:53:31.278 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ...
2024-05-17T14:53:31.313 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ...
2024-05-17T14:53:31.356 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-paste.
2024-05-17T14:53:31.382 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ...
2024-05-17T14:53:31.422 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ...
2024-05-17T14:53:31.629 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-simplegeneric.
2024-05-17T14:53:31.656 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ...
2024-05-17T14:53:31.695 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-simplegeneric (0.8.1-3) ...
2024-05-17T14:53:31.742 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python-pastedeploy-tpl.
2024-05-17T14:53:31.758 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ...
2024-05-17T14:53:31.800 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ...
2024-05-17T14:53:31.992 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-singledispatch.
2024-05-17T14:53:32.019 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ...
2024-05-17T14:53:32.067 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ...
2024-05-17T14:53:32.103 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pastedeploy.
2024-05-17T14:53:32.130 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ...
2024-05-17T14:53:32.170 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pastedeploy (2.1.1-1) ...
2024-05-17T14:53:32.349 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-webob.
2024-05-17T14:53:32.375 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1_all.deb ...
2024-05-17T14:53:32.414 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-webtest.
2024-05-17T14:53:32.415 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-webob (1:1.8.6-1.1) ...
2024-05-17T14:53:32.430 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ...
2024-05-17T14:53:32.465 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-webtest (2.0.35-1) ...
2024-05-17T14:53:32.759 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pecan.
2024-05-17T14:53:32.785 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ...
2024-05-17T14:53:32.795 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-waitress.
2024-05-17T14:53:32.822 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1_all.deb ...
2024-05-17T14:53:32.842 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ...
2024-05-17T14:53:32.866 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ...
2024-05-17T14:53:33.155 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-certifi.
2024-05-17T14:53:33.161 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-tempita.
2024-05-17T14:53:33.182 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../054-python3-certifi_2020.6.20-1_all.deb ...
2024-05-17T14:53:33.187 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ...
2024-05-17T14:53:33.214 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-certifi (2020.6.20-1) ...
2024-05-17T14:53:33.227 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ...
2024-05-17T14:53:33.526 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-paste.
2024-05-17T14:53:33.542 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-idna.
2024-05-17T14:53:33.553 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ...
2024-05-17T14:53:33.568 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../055-python3-idna_3.3-1_all.deb ...
2024-05-17T14:53:33.592 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ...
2024-05-17T14:53:33.608 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-idna (3.3-1) ...
2024-05-17T14:53:33.882 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python-pastedeploy-tpl.
2024-05-17T14:53:33.897 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ...
2024-05-17T14:53:33.919 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-urllib3.
2024-05-17T14:53:33.940 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ...
2024-05-17T14:53:33.946 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../056-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ...
2024-05-17T14:53:33.986 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ...
2024-05-17T14:53:34.247 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pastedeploy.
2024-05-17T14:53:34.273 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ...
2024-05-17T14:53:34.313 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pastedeploy (2.1.1-1) ...
2024-05-17T14:53:34.322 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-requests.
2024-05-17T14:53:34.349 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../057-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ...
2024-05-17T14:53:34.389 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ...
2024-05-17T14:53:34.612 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-webtest.
2024-05-17T14:53:34.638 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ...
2024-05-17T14:53:34.679 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-webtest (2.0.35-1) ...
2024-05-17T14:53:34.702 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-werkzeug.
2024-05-17T14:53:34.727 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../058-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ...
2024-05-17T14:53:34.767 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ...
2024-05-17T14:53:34.975 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pecan.
2024-05-17T14:53:35.002 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ...
2024-05-17T14:53:35.042 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ...
2024-05-17T14:53:35.061 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mgr-modules-core.
2024-05-17T14:53:35.077 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../059-ceph-mgr-modules-core_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:35.120 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mgr-modules-core (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:35.395 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-certifi.
2024-05-17T14:53:35.421 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../054-python3-certifi_2020.6.20-1_all.deb ...
2024-05-17T14:53:35.460 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-certifi (2020.6.20-1) ...
2024-05-17T14:53:35.500 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libsqlite3-mod-ceph.
2024-05-17T14:53:35.526 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../060-libsqlite3-mod-ceph_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:35.566 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libsqlite3-mod-ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:35.784 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-idna.
2024-05-17T14:53:35.812 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../055-python3-idna_3.3-1_all.deb ...
2024-05-17T14:53:35.851 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-idna (3.3-1) ...
2024-05-17T14:53:35.878 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-bcrypt.
2024-05-17T14:53:35.904 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../061-python3-bcrypt_3.2.0-1build1_amd64.deb ...
2024-05-17T14:53:35.944 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ...
2024-05-17T14:53:36.239 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-openssl.
2024-05-17T14:53:36.265 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../062-python3-openssl_21.0.0-1_all.deb ...
2024-05-17T14:53:36.300 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-urllib3.
2024-05-17T14:53:36.305 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-openssl (21.0.0-1) ...
2024-05-17T14:53:36.326 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../056-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ...
2024-05-17T14:53:36.366 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ...
2024-05-17T14:53:36.575 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mgr.
2024-05-17T14:53:36.601 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../063-ceph-mgr_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:36.641 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mgr (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:36.655 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-requests.
2024-05-17T14:53:36.671 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../057-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ...
2024-05-17T14:53:36.705 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ...
2024-05-17T14:53:37.004 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-werkzeug.
2024-05-17T14:53:37.031 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../058-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ...
2024-05-17T14:53:37.061 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mon.
2024-05-17T14:53:37.070 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ...
2024-05-17T14:53:37.088 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../064-ceph-mon_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:37.128 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mon (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:37.352 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mgr-modules-core.
2024-05-17T14:53:37.379 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../059-ceph-mgr-modules-core_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:37.427 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mgr-modules-core (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:37.748 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-osd.
2024-05-17T14:53:37.775 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../065-ceph-osd_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:37.804 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libsqlite3-mod-ceph.
2024-05-17T14:53:37.814 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-osd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:37.831 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../060-libsqlite3-mod-ceph_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:37.871 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libsqlite3-mod-ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:38.184 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-bcrypt.
2024-05-17T14:53:38.211 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../061-python3-bcrypt_3.2.0-1build1_amd64.deb ...
2024-05-17T14:53:38.251 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ...
2024-05-17T14:53:38.533 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-openssl.
2024-05-17T14:53:38.560 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../062-python3-openssl_21.0.0-1_all.deb ...
2024-05-17T14:53:38.599 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-openssl (21.0.0-1) ...
2024-05-17T14:53:38.818 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph.
2024-05-17T14:53:38.833 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../066-ceph_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:38.868 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:38.938 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mgr.
2024-05-17T14:53:38.964 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../063-ceph-mgr_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:39.006 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mgr (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:39.212 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-fuse.
2024-05-17T14:53:39.239 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../067-ceph-fuse_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:39.278 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:39.420 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mon.
2024-05-17T14:53:39.447 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../064-ceph-mon_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:39.487 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mon (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:39.640 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mds.
2024-05-17T14:53:39.667 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../068-ceph-mds_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:39.707 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mds (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:40.060 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package cephadm.
2024-05-17T14:53:40.101 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../069-cephadm_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:40.120 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:40.154 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-osd.
2024-05-17T14:53:40.182 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../065-ceph-osd_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:40.220 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-osd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:40.431 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-asyncssh.
2024-05-17T14:53:40.456 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../070-python3-asyncssh_2.5.0-1_all.deb ...
2024-05-17T14:53:40.498 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-asyncssh (2.5.0-1) ...
2024-05-17T14:53:40.949 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python-babel-localedata.
2024-05-17T14:53:40.975 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../071-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ...
2024-05-17T14:53:41.015 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ...
2024-05-17T14:53:41.694 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph.
2024-05-17T14:53:41.709 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../066-ceph_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:41.743 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:42.036 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-babel.
2024-05-17T14:53:42.040 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-fuse.
2024-05-17T14:53:42.063 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../072-python3-babel_2.8.0+dfsg.1-7_all.deb ...
2024-05-17T14:53:42.066 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../067-ceph-fuse_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:42.103 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ...
2024-05-17T14:53:42.106 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:42.423 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-jinja2.
2024-05-17T14:53:42.449 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../073-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ...
2024-05-17T14:53:42.489 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ...
2024-05-17T14:53:42.505 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mds.
2024-05-17T14:53:42.532 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../068-ceph-mds_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:42.572 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mds (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:42.850 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mgr-cephadm.
2024-05-17T14:53:42.877 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../074-ceph-mgr-cephadm_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:42.917 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mgr-cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:43.045 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package cephadm.
2024-05-17T14:53:43.074 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../069-cephadm_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:43.120 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:43.270 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-repoze.lru.
2024-05-17T14:53:43.297 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../075-python3-repoze.lru_0.7-2_all.deb ...
2024-05-17T14:53:43.337 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-repoze.lru (0.7-2) ...
2024-05-17T14:53:43.417 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-asyncssh.
2024-05-17T14:53:43.444 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../070-python3-asyncssh_2.5.0-1_all.deb ...
2024-05-17T14:53:43.484 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-asyncssh (2.5.0-1) ...
2024-05-17T14:53:43.640 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-routes.
2024-05-17T14:53:43.667 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../076-python3-routes_2.5.1-1ubuntu1_all.deb ...
2024-05-17T14:53:43.706 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ...
2024-05-17T14:53:43.773 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python-babel-localedata.
2024-05-17T14:53:43.800 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../071-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ...
2024-05-17T14:53:43.840 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ...
2024-05-17T14:53:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mgr-dashboard.
2024-05-17T14:53:44.070 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../077-ceph-mgr-dashboard_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:44.109 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mgr-dashboard (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:44.859 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-babel.
2024-05-17T14:53:44.886 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../072-python3-babel_2.8.0+dfsg.1-7_all.deb ...
2024-05-17T14:53:44.925 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ...
2024-05-17T14:53:45.267 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-jinja2.
2024-05-17T14:53:45.294 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../073-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ...
2024-05-17T14:53:45.341 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ...
2024-05-17T14:53:45.656 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-sklearn-lib:amd64.
2024-05-17T14:53:45.672 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../078-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ...
2024-05-17T14:53:45.680 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mgr-cephadm.
2024-05-17T14:53:45.706 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ...
2024-05-17T14:53:45.706 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../074-ceph-mgr-cephadm_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:45.746 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mgr-cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:46.079 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-repoze.lru.
2024-05-17T14:53:46.106 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../075-python3-repoze.lru_0.7-2_all.deb ...
2024-05-17T14:53:46.145 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-repoze.lru (0.7-2) ...
2024-05-17T14:53:46.187 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-joblib.
2024-05-17T14:53:46.200 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../079-python3-joblib_0.17.0-4ubuntu1_all.deb ...
2024-05-17T14:53:46.234 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ...
2024-05-17T14:53:46.596 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-threadpoolctl.
2024-05-17T14:53:46.623 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../080-python3-threadpoolctl_3.1.0-1_all.deb ...
2024-05-17T14:53:46.662 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ...
2024-05-17T14:53:46.842 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-routes.
2024-05-17T14:53:46.869 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../076-python3-routes_2.5.1-1ubuntu1_all.deb ...
2024-05-17T14:53:46.910 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ...
2024-05-17T14:53:46.965 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-sklearn.
2024-05-17T14:53:46.992 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../081-python3-sklearn_0.23.2-5ubuntu6_all.deb ...
2024-05-17T14:53:47.023 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ...
2024-05-17T14:53:47.233 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mgr-dashboard.
2024-05-17T14:53:47.259 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../077-ceph-mgr-dashboard_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:47.299 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mgr-dashboard (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:47.782 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local.
2024-05-17T14:53:47.810 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../082-ceph-mgr-diskprediction-local_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:47.934 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mgr-diskprediction-local (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:48.896 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-cachetools.
2024-05-17T14:53:48.923 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../083-python3-cachetools_5.0.0-1_all.deb ...
2024-05-17T14:53:48.943 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-sklearn-lib:amd64.
2024-05-17T14:53:48.959 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../078-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ...
2024-05-17T14:53:48.962 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-cachetools (5.0.0-1) ...
2024-05-17T14:53:49.002 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ...
2024-05-17T14:53:49.249 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pyasn1.
2024-05-17T14:53:49.276 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../084-python3-pyasn1_0.4.8-1_all.deb ...
2024-05-17T14:53:49.315 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pyasn1 (0.4.8-1) ...
2024-05-17T14:53:49.451 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-joblib.
2024-05-17T14:53:49.471 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../079-python3-joblib_0.17.0-4ubuntu1_all.deb ...
2024-05-17T14:53:49.509 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ...
2024-05-17T14:53:49.593 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pyasn1-modules.
2024-05-17T14:53:49.621 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../085-python3-pyasn1-modules_0.2.1-1_all.deb ...
2024-05-17T14:53:49.652 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ...
2024-05-17T14:53:49.859 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-threadpoolctl.
2024-05-17T14:53:49.886 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../080-python3-threadpoolctl_3.1.0-1_all.deb ...
2024-05-17T14:53:49.913 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-rsa.
2024-05-17T14:53:49.926 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../086-python3-rsa_4.8-1_all.deb ...
2024-05-17T14:53:49.934 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ...
2024-05-17T14:53:49.954 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-rsa (4.8-1) ...
2024-05-17T14:53:50.214 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-sklearn.
2024-05-17T14:53:50.242 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../081-python3-sklearn_0.23.2-5ubuntu6_all.deb ...
2024-05-17T14:53:50.280 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ...
2024-05-17T14:53:50.285 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-google-auth.
2024-05-17T14:53:50.312 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../087-python3-google-auth_1.5.1-3_all.deb ...
2024-05-17T14:53:50.352 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-google-auth (1.5.1-3) ...
2024-05-17T14:53:50.705 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-requests-oauthlib.
2024-05-17T14:53:50.732 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../088-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ...
2024-05-17T14:53:50.771 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ...
2024-05-17T14:53:50.891 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local.
2024-05-17T14:53:50.919 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../082-ceph-mgr-diskprediction-local_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:51.055 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mgr-diskprediction-local (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:51.083 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-websocket.
2024-05-17T14:53:51.110 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../089-python3-websocket_1.2.3-1_all.deb ...
2024-05-17T14:53:51.149 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-websocket (1.2.3-1) ...
2024-05-17T14:53:51.444 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-kubernetes.
2024-05-17T14:53:51.460 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../090-python3-kubernetes_12.0.1-1ubuntu1_all.deb ...
2024-05-17T14:53:51.494 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ...
2024-05-17T14:53:52.150 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-mgr-k8sevents.
2024-05-17T14:53:52.166 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../091-ceph-mgr-k8sevents_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:52.208 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-mgr-k8sevents (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:52.496 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-cachetools.
2024-05-17T14:53:52.524 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../083-python3-cachetools_5.0.0-1_all.deb ...
2024-05-17T14:53:52.536 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libonig5:amd64.
2024-05-17T14:53:52.563 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-cachetools (5.0.0-1) ...
2024-05-17T14:53:52.563 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../092-libonig5_6.9.7.1-2build1_amd64.deb ...
2024-05-17T14:53:52.602 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ...
2024-05-17T14:53:52.893 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pyasn1.
2024-05-17T14:53:52.896 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libjq1:amd64.
2024-05-17T14:53:52.909 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../084-python3-pyasn1_0.4.8-1_all.deb ...
2024-05-17T14:53:52.912 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../093-libjq1_1.6-2.1ubuntu3_amd64.deb ...
2024-05-17T14:53:52.946 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ...
2024-05-17T14:53:52.951 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pyasn1 (0.4.8-1) ...
2024-05-17T14:53:53.208 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package jq.
2024-05-17T14:53:53.235 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../094-jq_1.6-2.1ubuntu3_amd64.deb ...
2024-05-17T14:53:53.274 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking jq (1.6-2.1ubuntu3) ...
2024-05-17T14:53:53.275 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pyasn1-modules.
2024-05-17T14:53:53.303 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../085-python3-pyasn1-modules_0.2.1-1_all.deb ...
2024-05-17T14:53:53.342 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ...
2024-05-17T14:53:53.526 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package socat.
2024-05-17T14:53:53.542 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../095-socat_1.7.4.1-3ubuntu4_amd64.deb ...
2024-05-17T14:53:53.585 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ...
2024-05-17T14:53:53.632 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-rsa.
2024-05-17T14:53:53.659 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../086-python3-rsa_4.8-1_all.deb ...
2024-05-17T14:53:53.699 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-rsa (4.8-1) ...
2024-05-17T14:53:53.904 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package xmlstarlet.
2024-05-17T14:53:53.921 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../096-xmlstarlet_1.6.1-2.1_amd64.deb ...
2024-05-17T14:53:53.963 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking xmlstarlet (1.6.1-2.1) ...
2024-05-17T14:53:54.014 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-google-auth.
2024-05-17T14:53:54.041 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../087-python3-google-auth_1.5.1-3_all.deb ...
2024-05-17T14:53:54.089 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-google-auth (1.5.1-3) ...
2024-05-17T14:53:54.351 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-test.
2024-05-17T14:53:54.378 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../097-ceph-test_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:54.396 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-requests-oauthlib.
2024-05-17T14:53:54.424 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../088-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ...
2024-05-17T14:53:54.425 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-test (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:54.463 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ...
2024-05-17T14:53:54.762 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-websocket.
2024-05-17T14:53:54.789 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../089-python3-websocket_1.2.3-1_all.deb ...
2024-05-17T14:53:54.837 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-websocket (1.2.3-1) ...
2024-05-17T14:53:55.159 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-kubernetes.
2024-05-17T14:53:55.187 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../090-python3-kubernetes_12.0.1-1ubuntu1_all.deb ...
2024-05-17T14:53:55.226 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ...
2024-05-17T14:53:55.964 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-mgr-k8sevents.
2024-05-17T14:53:55.981 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../091-ceph-mgr-k8sevents_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:56.025 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-mgr-k8sevents (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:56.344 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libonig5:amd64.
2024-05-17T14:53:56.372 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../092-libonig5_6.9.7.1-2build1_amd64.deb ...
2024-05-17T14:53:56.411 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ...
2024-05-17T14:53:56.726 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libjq1:amd64.
2024-05-17T14:53:56.754 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../093-libjq1_1.6-2.1ubuntu3_amd64.deb ...
2024-05-17T14:53:57.027 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ...
2024-05-17T14:53:57.046 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package ceph-volume.
2024-05-17T14:53:57.073 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../098-ceph-volume_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:53:57.121 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking ceph-volume (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:57.289 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package jq.
2024-05-17T14:53:57.317 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../094-jq_1.6-2.1ubuntu3_amd64.deb ...
2024-05-17T14:53:57.358 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking jq (1.6-2.1ubuntu3) ...
2024-05-17T14:53:57.458 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package libcephfs-dev.
2024-05-17T14:53:57.474 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../099-libcephfs-dev_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:57.516 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking libcephfs-dev (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:57.612 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package socat.
2024-05-17T14:53:57.640 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../095-socat_1.7.4.1-3ubuntu4_amd64.deb ...
2024-05-17T14:53:57.679 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ...
2024-05-17T14:53:57.794 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package lua-socket:amd64.
2024-05-17T14:53:57.810 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../100-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ...
2024-05-17T14:53:57.852 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ...
2024-05-17T14:53:57.986 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package xmlstarlet.
2024-05-17T14:53:58.014 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../096-xmlstarlet_1.6.1-2.1_amd64.deb ...
2024-05-17T14:53:58.053 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking xmlstarlet (1.6.1-2.1) ...
2024-05-17T14:53:58.148 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package lua-sec:amd64.
2024-05-17T14:53:58.176 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../101-lua-sec_1.0.2-1_amd64.deb ...
2024-05-17T14:53:58.214 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ...
2024-05-17T14:53:58.378 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-test.
2024-05-17T14:53:58.395 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../097-ceph-test_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:53:58.437 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-test (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:53:58.602 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package nvme-cli.
2024-05-17T14:53:58.629 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../102-nvme-cli_1.16-3ubuntu0.1_amd64.deb ...
2024-05-17T14:53:58.668 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ...
2024-05-17T14:53:59.099 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package pkg-config.
2024-05-17T14:53:59.126 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../103-pkg-config_0.29.2-1ubuntu3_amd64.deb ...
2024-05-17T14:53:59.165 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ...
2024-05-17T14:53:59.418 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python-asyncssh-doc.
2024-05-17T14:53:59.434 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../104-python-asyncssh-doc_2.5.0-1_all.deb ...
2024-05-17T14:53:59.467 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ...
2024-05-17T14:53:59.915 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-iniconfig.
2024-05-17T14:53:59.940 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../105-python3-iniconfig_1.1.1-2_all.deb ...
2024-05-17T14:53:59.979 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-iniconfig (1.1.1-2) ...
2024-05-17T14:54:00.248 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pastescript.
2024-05-17T14:54:00.276 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../106-python3-pastescript_2.0.2-4_all.deb ...
2024-05-17T14:54:00.306 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pastescript (2.0.2-4) ...
2024-05-17T14:54:00.610 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pluggy.
2024-05-17T14:54:00.637 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../107-python3-pluggy_0.13.0-7.1_all.deb ...
2024-05-17T14:54:00.676 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pluggy (0.13.0-7.1) ...
2024-05-17T14:54:00.971 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-psutil.
2024-05-17T14:54:00.998 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../108-python3-psutil_5.9.0-1build1_amd64.deb ...
2024-05-17T14:54:01.037 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-psutil (5.9.0-1build1) ...
2024-05-17T14:54:01.348 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-py.
2024-05-17T14:54:01.376 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../109-python3-py_1.10.0-1_all.deb ...
2024-05-17T14:54:01.414 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-py (1.10.0-1) ...
2024-05-17T14:54:01.743 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pygments.
2024-05-17T14:54:01.771 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../110-python3-pygments_2.11.2+dfsg-2_all.deb ...
2024-05-17T14:54:01.810 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ...
2024-05-17T14:54:01.988 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package ceph-volume.
2024-05-17T14:54:02.015 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../098-ceph-volume_19.0.0-3735-gee59d86a-1jammy_all.deb ...
2024-05-17T14:54:02.054 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking ceph-volume (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:02.239 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pyinotify.
2024-05-17T14:54:02.255 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../111-python3-pyinotify_0.9.6-1.3_all.deb ...
2024-05-17T14:54:02.297 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ...
2024-05-17T14:54:02.353 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package libcephfs-dev.
2024-05-17T14:54:02.369 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../099-libcephfs-dev_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:02.455 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking libcephfs-dev (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:02.591 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-toml.
2024-05-17T14:54:02.619 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../112-python3-toml_0.10.2-1_all.deb ...
2024-05-17T14:54:02.657 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-toml (0.10.2-1) ...
2024-05-17T14:54:02.777 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package lua-socket:amd64.
2024-05-17T14:54:02.805 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../100-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ...
2024-05-17T14:54:02.852 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ...
2024-05-17T14:54:02.960 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-pytest.
2024-05-17T14:54:02.989 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../113-python3-pytest_6.2.5-1ubuntu2_all.deb ...
2024-05-17T14:54:03.027 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ...
2024-05-17T14:54:03.187 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package lua-sec:amd64.
2024-05-17T14:54:03.214 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../101-lua-sec_1.0.2-1_amd64.deb ...
2024-05-17T14:54:03.253 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ...
2024-05-17T14:54:03.372 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package python3-simplejson.
2024-05-17T14:54:03.400 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../114-python3-simplejson_3.17.6-1build1_amd64.deb ...
2024-05-17T14:54:03.438 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking python3-simplejson (3.17.6-1build1) ...
2024-05-17T14:54:03.612 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package nvme-cli.
2024-05-17T14:54:03.639 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../102-nvme-cli_1.16-3ubuntu0.1_amd64.deb ...
2024-05-17T14:54:03.678 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ...
2024-05-17T14:54:03.699 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package qttranslations5-l10n.
2024-05-17T14:54:03.727 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../115-qttranslations5-l10n_5.15.3-1_all.deb ...
2024-05-17T14:54:03.766 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ...
2024-05-17T14:54:04.073 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package pkg-config.
2024-05-17T14:54:04.100 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../103-pkg-config_0.29.2-1ubuntu3_amd64.deb ...
2024-05-17T14:54:04.139 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ...
2024-05-17T14:54:04.396 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python-asyncssh-doc.
2024-05-17T14:54:04.412 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../104-python-asyncssh-doc_2.5.0-1_all.deb ...
2024-05-17T14:54:04.445 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package radosgw.
2024-05-17T14:54:04.446 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ...
2024-05-17T14:54:04.473 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../116-radosgw_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:04.511 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking radosgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:04.852 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-iniconfig.
2024-05-17T14:54:04.879 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../105-python3-iniconfig_1.1.1-2_all.deb ...
2024-05-17T14:54:04.918 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-iniconfig (1.1.1-2) ...
2024-05-17T14:54:05.208 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pastescript.
2024-05-17T14:54:05.238 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../106-python3-pastescript_2.0.2-4_all.deb ...
2024-05-17T14:54:05.283 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pastescript (2.0.2-4) ...
2024-05-17T14:54:05.406 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package rbd-fuse.
2024-05-17T14:54:05.434 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../117-rbd-fuse_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:05.472 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking rbd-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:05.590 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pluggy.
2024-05-17T14:54:05.618 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../107-python3-pluggy_0.13.0-7.1_all.deb ...
2024-05-17T14:54:05.657 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pluggy (0.13.0-7.1) ...
2024-05-17T14:54:05.825 INFO:teuthology.orchestra.run.smithi171.stdout:Selecting previously unselected package rbd-mirror.
2024-05-17T14:54:05.854 INFO:teuthology.orchestra.run.smithi171.stdout:Preparing to unpack .../118-rbd-mirror_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:05.891 INFO:teuthology.orchestra.run.smithi171.stdout:Unpacking rbd-mirror (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:05.945 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-psutil.
2024-05-17T14:54:05.973 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../108-python3-psutil_5.9.0-1build1_amd64.deb ...
2024-05-17T14:54:06.012 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-psutil (5.9.0-1build1) ...
2024-05-17T14:54:06.342 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-iniconfig (1.1.1-2) ...
2024-05-17T14:54:06.344 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-py.
2024-05-17T14:54:06.371 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../109-python3-py_1.10.0-1_all.deb ...
2024-05-17T14:54:06.410 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-py (1.10.0-1) ...
2024-05-17T14:54:06.605 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ...
2024-05-17T14:54:06.731 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ...
2024-05-17T14:54:06.768 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pygments.
2024-05-17T14:54:06.795 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../110-python3-pygments_2.11.2+dfsg-2_all.deb ...
2024-05-17T14:54:06.835 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ...
2024-05-17T14:54:06.977 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service.
2024-05-17T14:54:07.293 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service.
2024-05-17T14:54:07.435 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pyinotify.
2024-05-17T14:54:07.462 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../111-python3-pyinotify_0.9.6-1.3_all.deb ...
2024-05-17T14:54:07.643 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ...
2024-05-17T14:54:07.729 INFO:teuthology.orchestra.run.smithi171.stdout:nvmf-connect.target is a disabled or a static unit, not starting it.
2024-05-17T14:54:07.816 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:07.931 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-toml.
2024-05-17T14:54:07.959 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../112-python3-toml_0.10.2-1_all.deb ...
2024-05-17T14:54:07.998 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-toml (0.10.2-1) ...
2024-05-17T14:54:08.296 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-pytest.
2024-05-17T14:54:08.324 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../113-python3-pytest_6.2.5-1ubuntu2_all.deb ...
2024-05-17T14:54:08.340 INFO:teuthology.orchestra.run.smithi171.stdout:Adding system user cephadm....done
2024-05-17T14:54:08.371 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ...
2024-05-17T14:54:08.496 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ...
2024-05-17T14:54:08.702 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package python3-simplejson.
2024-05-17T14:54:08.730 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../114-python3-simplejson_3.17.6-1build1_amd64.deb ...
2024-05-17T14:54:08.765 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-jaraco.classes (3.2.1-3) ...
2024-05-17T14:54:08.769 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking python3-simplejson (3.17.6-1build1) ...
2024-05-17T14:54:09.024 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python-asyncssh-doc (2.5.0-1) ...
2024-05-17T14:54:09.033 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package qttranslations5-l10n.
2024-05-17T14:54:09.061 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../115-qttranslations5-l10n_5.15.3-1_all.deb ...
2024-05-17T14:54:09.100 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ...
2024-05-17T14:54:09.150 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-jaraco.functools (3.4.0-2) ...
2024-05-17T14:54:09.393 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-repoze.lru (0.7-2) ...
2024-05-17T14:54:09.636 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ...
2024-05-17T14:54:09.745 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-py (1.10.0-1) ...
2024-05-17T14:54:09.800 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package radosgw.
2024-05-17T14:54:09.828 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../116-radosgw_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:09.867 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking radosgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:10.038 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ...
2024-05-17T14:54:10.407 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-cachetools (5.0.0-1) ...
2024-05-17T14:54:10.667 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-openssl (21.0.0-1) ...
2024-05-17T14:54:10.926 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up unzip (6.0-26ubuntu3.2) ...
2024-05-17T14:54:10.981 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package rbd-fuse.
2024-05-17T14:54:10.997 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../117-rbd-fuse_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:11.031 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking rbd-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:11.186 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-bcrypt (3.2.0-1build1) ...
2024-05-17T14:54:11.371 INFO:teuthology.orchestra.run.smithi072.stdout:Selecting previously unselected package rbd-mirror.
2024-05-17T14:54:11.399 INFO:teuthology.orchestra.run.smithi072.stdout:Preparing to unpack .../118-rbd-mirror_19.0.0-3735-gee59d86a-1jammy_amd64.deb ...
2024-05-17T14:54:11.428 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pyinotify (0.9.6-1.3) ...
2024-05-17T14:54:11.437 INFO:teuthology.orchestra.run.smithi072.stdout:Unpacking rbd-mirror (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:11.696 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-threadpoolctl (3.1.0-1) ...
2024-05-17T14:54:11.939 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-ceph-argparse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:11.971 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-iniconfig (1.1.1-2) ...
2024-05-17T14:54:12.224 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ...
2024-05-17T14:54:12.278 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ...
2024-05-17T14:54:12.344 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ...
2024-05-17T14:54:12.491 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ...
2024-05-17T14:54:12.593 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service.
2024-05-17T14:54:12.617 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ...
2024-05-17T14:54:12.743 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-markupsafe (2.0.1-2build1) ...
2024-05-17T14:54:12.973 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service.
2024-05-17T14:54:13.003 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up lua5.1 (5.1.5-8.1build4) ...
2024-05-17T14:54:13.365 INFO:teuthology.orchestra.run.smithi171.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode
2024-05-17T14:54:13.393 INFO:teuthology.orchestra.run.smithi171.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode
2024-05-17T14:54:13.402 INFO:teuthology.orchestra.run.smithi072.stdout:nvmf-connect.target is a disabled or a static unit, not starting it.
2024-05-17T14:54:13.456 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ...
2024-05-17T14:54:13.464 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:13.590 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-psutil (5.9.0-1build1) ...
2024-05-17T14:54:13.958 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ...
2024-05-17T14:54:13.978 INFO:teuthology.orchestra.run.smithi072.stdout:Adding system user cephadm....done
2024-05-17T14:54:14.076 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-natsort (8.0.2-1) ...
2024-05-17T14:54:14.136 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ...
2024-05-17T14:54:14.318 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ...
2024-05-17T14:54:14.398 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-jaraco.classes (3.2.1-3) ...
2024-05-17T14:54:14.586 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-simplejson (3.17.6-1build1) ...
2024-05-17T14:54:14.651 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python-asyncssh-doc (2.5.0-1) ...
2024-05-17T14:54:14.778 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-jaraco.functools (3.4.0-2) ...
2024-05-17T14:54:14.887 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up zip (3.0-12build2) ...
2024-05-17T14:54:15.013 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ...
2024-05-17T14:54:15.015 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-repoze.lru (0.7-2) ...
2024-05-17T14:54:15.275 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ...
2024-05-17T14:54:15.402 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-py (1.10.0-1) ...
2024-05-17T14:54:15.665 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ...
2024-05-17T14:54:15.706 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ...
2024-05-17T14:54:15.925 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ...
2024-05-17T14:54:16.274 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-cachetools (5.0.0-1) ...
2024-05-17T14:54:16.275 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up qttranslations5-l10n (5.15.3-1) ...
2024-05-17T14:54:16.275 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ...
2024-05-17T14:54:16.321 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-openssl (21.0.0-1) ...
2024-05-17T14:54:16.428 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-asyncssh (2.5.0-1) ...
2024-05-17T14:54:16.590 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up unzip (6.0-26ubuntu3.2) ...
2024-05-17T14:54:16.812 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-certifi (2020.6.20-1) ...
2024-05-17T14:54:16.836 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-bcrypt (3.2.0-1build1) ...
2024-05-17T14:54:17.058 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ...
2024-05-17T14:54:17.088 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pyinotify (0.9.6-1.3) ...
2024-05-17T14:54:17.340 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-threadpoolctl (3.1.0-1) ...
2024-05-17T14:54:17.448 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ...
2024-05-17T14:54:17.592 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-ceph-argparse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:17.741 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ...
2024-05-17T14:54:17.929 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ...
2024-05-17T14:54:18.056 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ...
2024-05-17T14:54:18.092 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-jaraco.text (3.6.0-2) ...
2024-05-17T14:54:18.166 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ...
2024-05-17T14:54:18.293 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-markupsafe (2.0.1-2build1) ...
2024-05-17T14:54:18.327 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up socat (1.7.4.1-3ubuntu4) ...
2024-05-17T14:54:18.436 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:18.554 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up lua5.1 (5.1.5-8.1build4) ...
2024-05-17T14:54:18.721 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-idna (3.3-1) ...
2024-05-17T14:54:18.954 INFO:teuthology.orchestra.run.smithi072.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode
2024-05-17T14:54:18.981 INFO:teuthology.orchestra.run.smithi072.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode
2024-05-17T14:54:19.047 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ...
2024-05-17T14:54:19.052 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ...
2024-05-17T14:54:19.181 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-psutil (5.9.0-1build1) ...
2024-05-17T14:54:19.541 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ...
2024-05-17T14:54:19.668 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-natsort (8.0.2-1) ...
2024-05-17T14:54:19.929 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ...
2024-05-17T14:54:20.182 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-simplejson (3.17.6-1build1) ...
2024-05-17T14:54:20.244 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ...
2024-05-17T14:54:20.462 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:54:20.484 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up zip (3.0-12build2) ...
2024-05-17T14:54:20.590 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-toml (0.10.2-1) ...
2024-05-17T14:54:20.603 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ...
2024-05-17T14:54:20.889 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ...
2024-05-17T14:54:21.016 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up xmlstarlet (1.6.1-2.1) ...
2024-05-17T14:54:21.142 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ...
2024-05-17T14:54:21.292 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ...
2024-05-17T14:54:21.435 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pluggy (0.13.0-7.1) ...
2024-05-17T14:54:21.560 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ...
2024-05-17T14:54:21.671 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up qttranslations5-l10n (5.15.3-1) ...
2024-05-17T14:54:21.678 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-zc.lockfile (2.0-1) ...
2024-05-17T14:54:21.797 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ...
2024-05-17T14:54:21.920 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:54:22.030 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pyasn1 (0.4.8-1) ...
2024-05-17T14:54:22.091 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-asyncssh (2.5.0-1) ...
2024-05-17T14:54:22.331 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-singledispatch (3.4.0.3-3) ...
2024-05-17T14:54:22.487 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-certifi (2020.6.20-1) ...
2024-05-17T14:54:22.590 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-logutils (0.3.3-8) ...
2024-05-17T14:54:22.722 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ...
2024-05-17T14:54:22.850 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-tempora (4.1.2-1) ...
2024-05-17T14:54:23.093 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-simplegeneric (0.8.1-3) ...
2024-05-17T14:54:23.100 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ...
2024-05-17T14:54:23.344 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-prettytable (2.5.0-2) ...
2024-05-17T14:54:23.387 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ...
2024-05-17T14:54:23.595 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ...
2024-05-17T14:54:23.696 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-websocket (1.2.3-1) ...
2024-05-17T14:54:23.732 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-jaraco.text (3.6.0-2) ...
2024-05-17T14:54:23.967 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up socat (1.7.4.1-3ubuntu4) ...
2024-05-17T14:54:23.971 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ...
2024-05-17T14:54:24.094 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:24.098 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ...
2024-05-17T14:54:24.381 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-idna (3.3-1) ...
2024-05-17T14:54:24.391 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-webob (1:1.8.6-1.1) ...
2024-05-17T14:54:24.683 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ...
2024-05-17T14:54:24.683 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-jaraco.collections (3.4.0-2) ...
2024-05-17T14:54:24.927 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ...
2024-05-17T14:54:25.060 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up lua-sec:amd64 (1.0.2-1) ...
2024-05-17T14:54:25.187 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ...
2024-05-17T14:54:25.293 INFO:teuthology.orchestra.run.smithi171.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode
2024-05-17T14:54:25.530 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ...
2024-05-17T14:54:25.656 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ...
2024-05-17T14:54:25.887 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ...
2024-05-17T14:54:26.049 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pastedeploy (2.1.1-1) ...
2024-05-17T14:54:26.142 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:54:26.277 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-toml (0.10.2-1) ...
2024-05-17T14:54:26.300 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up lua-any (27ubuntu1) ...
2024-05-17T14:54:26.426 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-portend (3.0.0-1) ...
2024-05-17T14:54:26.529 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ...
2024-05-17T14:54:26.656 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up xmlstarlet (1.6.1-2.1) ...
2024-05-17T14:54:26.669 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:54:26.770 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ...
2024-05-17T14:54:26.776 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ...
2024-05-17T14:54:27.071 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ...
2024-05-17T14:54:27.095 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pluggy (0.13.0-7.1) ...
2024-05-17T14:54:27.355 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-zc.lockfile (2.0-1) ...
2024-05-17T14:54:27.372 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ...
2024-05-17T14:54:27.607 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:54:27.640 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up jq (1.6-2.1ubuntu3) ...
2024-05-17T14:54:27.736 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pyasn1 (0.4.8-1) ...
2024-05-17T14:54:27.766 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-rsa (4.8-1) ...
2024-05-17T14:54:28.030 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-singledispatch (3.4.0.3-3) ...
2024-05-17T14:54:28.034 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-webtest (2.0.35-1) ...
2024-05-17T14:54:28.293 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ...
2024-05-17T14:54:28.299 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-logutils (0.3.3-8) ...
2024-05-17T14:54:28.551 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-tempora (4.1.2-1) ...
2024-05-17T14:54:28.613 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-cherrypy3 (18.6.1-4) ...
2024-05-17T14:54:28.804 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-simplegeneric (0.8.1-3) ...
2024-05-17T14:54:28.998 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pastescript (2.0.2-4) ...
2024-05-17T14:54:29.056 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-prettytable (2.5.0-2) ...
2024-05-17T14:54:29.283 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ...
2024-05-17T14:54:29.308 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ...
2024-05-17T14:54:29.435 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-websocket (1.2.3-1) ...
2024-05-17T14:54:29.602 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ...
2024-05-17T14:54:29.714 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ...
2024-05-17T14:54:29.728 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up librados2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:29.833 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ...
2024-05-17T14:54:29.854 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libsqlite3-mod-ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:29.980 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up luarocks (3.8.0+dfsg1-1) ...
2024-05-17T14:54:30.118 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-webob (1:1.8.6-1.1) ...
2024-05-17T14:54:30.325 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libcephfs2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:30.420 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-jaraco.collections (3.4.0-2) ...
2024-05-17T14:54:30.451 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libradosstriper1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:30.577 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-google-auth (1.5.1-3) ...
2024-05-17T14:54:30.674 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ...
2024-05-17T14:54:30.801 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up lua-sec:amd64 (1.0.2-1) ...
2024-05-17T14:54:30.836 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up librbd1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:30.928 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ...
2024-05-17T14:54:30.962 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mgr-modules-core (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:31.018 INFO:teuthology.orchestra.run.smithi072.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode
2024-05-17T14:54:31.088 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:31.282 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ...
2024-05-17T14:54:31.301 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target.
2024-05-17T14:54:31.302 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target.
2024-05-17T14:54:31.402 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ...
2024-05-17T14:54:31.788 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pastedeploy (2.1.1-1) ...
2024-05-17T14:54:31.814 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up libcephfs-dev (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:31.966 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-rados (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:32.039 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up lua-any (27ubuntu1) ...
2024-05-17T14:54:32.083 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up librgw2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:32.184 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ...
2024-05-17T14:54:32.201 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-portend (3.0.0-1) ...
2024-05-17T14:54:32.461 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ...
2024-05-17T14:54:32.588 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ...
2024-05-17T14:54:32.898 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ...
2024-05-17T14:54:33.194 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ...
2024-05-17T14:54:33.346 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-rbd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:33.463 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up jq (1.6-2.1ubuntu3) ...
2024-05-17T14:54:33.472 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up rbd-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:33.590 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-rsa (4.8-1) ...
2024-05-17T14:54:33.598 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-rgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:33.749 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up python3-cephfs (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:33.858 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-webtest (2.0.35-1) ...
2024-05-17T14:54:33.867 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:34.137 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ...
2024-05-17T14:54:34.352 INFO:teuthology.orchestra.run.smithi171.stdout:Adding group ceph....done
2024-05-17T14:54:34.468 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-cherrypy3 (18.6.1-4) ...
2024-05-17T14:54:34.695 INFO:teuthology.orchestra.run.smithi171.stdout:Adding system user ceph....done
2024-05-17T14:54:34.855 INFO:teuthology.orchestra.run.smithi171.stdout:Setting system user ceph properties....done
2024-05-17T14:54:34.856 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pastescript (2.0.2-4) ...
2024-05-17T14:54:34.869 INFO:teuthology.orchestra.run.smithi171.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory
2024-05-17T14:54:35.001 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target.
2024-05-17T14:54:35.142 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ...
2024-05-17T14:54:35.321 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service.
2024-05-17T14:54:35.490 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ...
2024-05-17T14:54:35.609 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up librados2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:35.727 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libsqlite3-mod-ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:35.818 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-test (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:35.854 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up luarocks (3.8.0+dfsg1-1) ...
2024-05-17T14:54:35.919 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up radosgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:36.178 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libcephfs2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:36.305 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libradosstriper1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:36.399 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target.
2024-05-17T14:54:36.400 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target.
2024-05-17T14:54:36.432 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-google-auth (1.5.1-3) ...
2024-05-17T14:54:36.711 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up librbd1 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:36.821 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mgr-modules-core (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:36.879 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-base (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:36.948 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:37.162 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target.
2024-05-17T14:54:37.163 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target.
2024-05-17T14:54:37.235 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service.
2024-05-17T14:54:37.626 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up libcephfs-dev (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:37.731 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mds (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:37.754 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-rados (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:37.881 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up librgw2 (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:37.987 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target.
2024-05-17T14:54:37.987 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target.
2024-05-17T14:54:38.008 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ...
2024-05-17T14:54:38.474 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up rbd-mirror (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:38.682 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target.
2024-05-17T14:54:38.682 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target.
2024-05-17T14:54:39.142 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mgr (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:39.225 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-rbd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:39.348 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target.
2024-05-17T14:54:39.348 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target.
2024-05-17T14:54:39.352 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up rbd-fuse (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:39.479 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-rgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:39.607 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up python3-cephfs (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:39.734 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-common (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:39.827 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-osd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:40.096 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target.
2024-05-17T14:54:40.096 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target.
2024-05-17T14:54:40.266 INFO:teuthology.orchestra.run.smithi072.stdout:Adding group ceph....done
2024-05-17T14:54:40.537 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mgr-k8sevents (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:40.605 INFO:teuthology.orchestra.run.smithi072.stdout:Adding system user ceph....done
2024-05-17T14:54:40.663 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mgr-diskprediction-local (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:40.775 INFO:teuthology.orchestra.run.smithi072.stdout:Setting system user ceph properties....done
2024-05-17T14:54:40.792 INFO:teuthology.orchestra.run.smithi072.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory
2024-05-17T14:54:40.814 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mon (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:40.930 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target.
2024-05-17T14:54:41.047 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target.
2024-05-17T14:54:41.047 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target.
2024-05-17T14:54:41.270 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service.
2024-05-17T14:54:41.499 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mgr-cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:41.650 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:41.751 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-mgr-dashboard (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:41.763 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-test (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:41.877 INFO:teuthology.orchestra.run.smithi171.stdout:Setting up ceph-volume (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:41.890 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up radosgw (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:42.248 INFO:teuthology.orchestra.run.smithi171.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ...
2024-05-17T14:54:42.333 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target.
2024-05-17T14:54:42.333 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target.
2024-05-17T14:54:42.365 INFO:teuthology.orchestra.run.smithi171.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ...
2024-05-17T14:54:42.520 INFO:teuthology.orchestra.run.smithi171.stdout:Processing triggers for man-db (2.10.2-1) ...
2024-05-17T14:54:42.787 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-base (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:43.140 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service.
2024-05-17T14:54:43.598 INFO:teuthology.orchestra.run.smithi171.stdout:Processing triggers for install-info (6.8-4build1) ...
2024-05-17T14:54:43.624 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mds (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:43.873 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target.
2024-05-17T14:54:43.874 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target.
2024-05-17T14:54:44.243 INFO:teuthology.orchestra.run.smithi171.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead.
2024-05-17T14:54:44.249 DEBUG:teuthology.parallel:result is None
2024-05-17T14:54:44.337 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up rbd-mirror (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:44.550 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target.
2024-05-17T14:54:44.551 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target.
2024-05-17T14:54:44.998 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mgr (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:45.215 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target.
2024-05-17T14:54:45.215 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target.
2024-05-17T14:54:45.726 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-osd (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:45.998 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target.
2024-05-17T14:54:45.998 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target.
2024-05-17T14:54:46.505 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mgr-k8sevents (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:46.632 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mgr-diskprediction-local (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:46.767 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mon (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:46.988 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target.
2024-05-17T14:54:46.988 INFO:teuthology.orchestra.run.smithi072.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target.
2024-05-17T14:54:47.486 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mgr-cephadm (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:47.640 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:47.758 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-mgr-dashboard (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:47.893 INFO:teuthology.orchestra.run.smithi072.stdout:Setting up ceph-volume (19.0.0-3735-gee59d86a-1jammy) ...
2024-05-17T14:54:48.266 INFO:teuthology.orchestra.run.smithi072.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ...
2024-05-17T14:54:48.376 INFO:teuthology.orchestra.run.smithi072.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ...
2024-05-17T14:54:48.530 INFO:teuthology.orchestra.run.smithi072.stdout:Processing triggers for man-db (2.10.2-1) ...
2024-05-17T14:54:50.135 INFO:teuthology.orchestra.run.smithi072.stdout:Processing triggers for install-info (6.8-4build1) ...
2024-05-17T14:54:50.893 INFO:teuthology.orchestra.run.smithi072.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead.
2024-05-17T14:54:50.899 DEBUG:teuthology.parallel:result is None
2024-05-17T14:54:50.899 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:54:51.058 DEBUG:teuthology.orchestra.run.smithi072:> dpkg-query -W -f '${Version}' ceph
2024-05-17T14:54:51.077 INFO:teuthology.orchestra.run.smithi072.stdout:19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:54:51.078 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:54:51.078 INFO:teuthology.task.install:The correct ceph version 19.0.0-3735-gee59d86a-1jammy is installed.
2024-05-17T14:54:51.079 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:54:51.295 DEBUG:teuthology.orchestra.run.smithi171:> dpkg-query -W -f '${Version}' ceph
2024-05-17T14:54:51.314 INFO:teuthology.orchestra.run.smithi171.stdout:19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:54:51.314 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-3735-gee59d86a-1jammy
2024-05-17T14:54:51.315 INFO:teuthology.task.install:The correct ceph version 19.0.0-3735-gee59d86a-1jammy is installed.
2024-05-17T14:54:51.316 INFO:teuthology.task.install.util:Shipping valgrind.supp...
2024-05-17T14:54:51.317 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:51.317 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp
2024-05-17T14:54:51.330 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:54:51.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp
2024-05-17T14:54:51.369 INFO:teuthology.task.install.util:Shipping 'daemon-helper'...
2024-05-17T14:54:51.369 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:51.370 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/usr/bin/daemon-helper
2024-05-17T14:54:51.384 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod a=rx -- /usr/bin/daemon-helper
2024-05-17T14:54:51.439 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:54:51.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/daemon-helper
2024-05-17T14:54:51.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/daemon-helper
2024-05-17T14:54:51.508 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'...
2024-05-17T14:54:51.508 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:51.508 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/usr/bin/adjust-ulimits
2024-05-17T14:54:51.521 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod a=rx -- /usr/bin/adjust-ulimits
2024-05-17T14:54:51.576 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:54:51.577 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/adjust-ulimits
2024-05-17T14:54:51.589 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/adjust-ulimits
2024-05-17T14:54:51.643 INFO:teuthology.task.install.util:Shipping 'stdin-killer'...
2024-05-17T14:54:51.644 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:51.644 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/usr/bin/stdin-killer
2024-05-17T14:54:51.657 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod a=rx -- /usr/bin/stdin-killer
2024-05-17T14:54:51.712 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:54:51.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/stdin-killer
2024-05-17T14:54:51.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/stdin-killer
2024-05-17T14:54:51.779 INFO:teuthology.run_tasks:Running task ceph...
2024-05-17T14:54:51.864 INFO:tasks.ceph:Making ceph log dir writeable by non-root...
2024-05-17T14:54:51.864 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod 777 /var/log/ceph
2024-05-17T14:54:51.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 777 /var/log/ceph
2024-05-17T14:54:51.879 INFO:tasks.ceph:Disabling ceph logrotate...
2024-05-17T14:54:51.879 DEBUG:teuthology.orchestra.run.smithi072:> sudo rm -f -- /etc/logrotate.d/ceph
2024-05-17T14:54:51.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /etc/logrotate.d/ceph
2024-05-17T14:54:51.934 INFO:tasks.ceph:Creating extra log directories...
2024-05-17T14:54:51.934 DEBUG:teuthology.orchestra.run.smithi072:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger
2024-05-17T14:54:51.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger
2024-05-17T14:54:52.004 INFO:tasks.ceph:Creating ceph cluster cluster1...
2024-05-17T14:54:52.004 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True}
2024-05-17T14:54:52.005 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217', 'branch': 'wip-dis-testing', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'email': 'idryomov@gmail.com', 'first_in_suite': False, 'job_id': '7711217', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-dis-testing'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-dis-testing', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, 'owner': 'scheduled_dis@teuthology', 'priority': 70, 'repo': 'https://github.com/ceph/ceph-ci', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 3754, 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4', 'sleep_before_teardown': 0, 'subset': '56/99', 'suite': 'rbd', 'suite_branch': 'wip-dis-testing', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci', 'suite_sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4', 'targets': {'smithi072.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAtFxZM3PX3EqFG5Suxok8lQhwmW0wU9dQiMCx105AEPlaLR+Mmd/eylLsthBupi9TbcCp1qT5241HblZVZk0AQ=', 'smithi171.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG6ZXSZlCBgx+DF91uuAf3IuG4mnOVQsHYBVdotBdA7HEH0Zygg0Zvz4SN+EXfOn2SaSabD5rx9Bw8uZG/mFa8k='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'wip-fix-extra-system-packages', 'teuthology_sha1': 'cad7edd8d899a7c59a48f7da3294fca4cb778f64', 'timestamp': '2024-05-17_14:29:27', 'tube': 'smithi', 'user': 'dis', 'verbose': True, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885'}
2024-05-17T14:54:52.005 DEBUG:teuthology.orchestra.run.smithi072:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data
2024-05-17T14:54:52.035 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data
2024-05-17T14:54:52.042 DEBUG:teuthology.orchestra.run.smithi072:> sudo install -d -m0777 -- /var/run/ceph
2024-05-17T14:54:52.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo install -d -m0777 -- /var/run/ceph
2024-05-17T14:54:52.096 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:52.096 DEBUG:teuthology.orchestra.run.smithi072:> dd if=/scratch_devs of=/dev/stdout
2024-05-17T14:54:52.141 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4']
2024-05-17T14:54:52.141 DEBUG:teuthology.orchestra.run.smithi072:> stat /dev/vg_nvme/lv_1
2024-05-17T14:54:52.189 INFO:teuthology.orchestra.run.smithi072.stdout:  File: /dev/vg_nvme/lv_1 -> ../dm-0
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout:Device: 5h/5d	Inode: 870         Links: 1
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout:Access: 2024-05-17 14:49:18.674752128 +0000
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout:Modify: 2024-05-17 14:49:18.414754451 +0000
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout:Change: 2024-05-17 14:49:18.414754451 +0000
2024-05-17T14:54:52.190 INFO:teuthology.orchestra.run.smithi072.stdout: Birth: -
2024-05-17T14:54:52.190 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1
2024-05-17T14:54:52.247 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records in
2024-05-17T14:54:52.247 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records out
2024-05-17T14:54:52.248 INFO:teuthology.orchestra.run.smithi072.stderr:512 bytes copied, 0.000444923 s, 1.2 MB/s
2024-05-17T14:54:52.249 DEBUG:teuthology.orchestra.run.smithi072:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1
2024-05-17T14:54:52.298 DEBUG:teuthology.orchestra.run.smithi072:> stat /dev/vg_nvme/lv_2
2024-05-17T14:54:52.349 INFO:teuthology.orchestra.run.smithi072.stdout:  File: /dev/vg_nvme/lv_2 -> ../dm-1
2024-05-17T14:54:52.349 INFO:teuthology.orchestra.run.smithi072.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:54:52.349 INFO:teuthology.orchestra.run.smithi072.stdout:Device: 5h/5d	Inode: 889         Links: 1
2024-05-17T14:54:52.350 INFO:teuthology.orchestra.run.smithi072.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:54:52.350 INFO:teuthology.orchestra.run.smithi072.stdout:Access: 2024-05-17 14:49:19.146747912 +0000
2024-05-17T14:54:52.350 INFO:teuthology.orchestra.run.smithi072.stdout:Modify: 2024-05-17 14:49:18.898750127 +0000
2024-05-17T14:54:52.350 INFO:teuthology.orchestra.run.smithi072.stdout:Change: 2024-05-17 14:49:18.898750127 +0000
2024-05-17T14:54:52.350 INFO:teuthology.orchestra.run.smithi072.stdout: Birth: -
2024-05-17T14:54:52.350 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1
2024-05-17T14:54:52.403 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records in
2024-05-17T14:54:52.403 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records out
2024-05-17T14:54:52.403 INFO:teuthology.orchestra.run.smithi072.stderr:512 bytes copied, 0.000344376 s, 1.5 MB/s
2024-05-17T14:54:52.404 DEBUG:teuthology.orchestra.run.smithi072:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2
2024-05-17T14:54:52.454 DEBUG:teuthology.orchestra.run.smithi072:> stat /dev/vg_nvme/lv_3
2024-05-17T14:54:52.501 INFO:teuthology.orchestra.run.smithi072.stdout:  File: /dev/vg_nvme/lv_3 -> ../dm-2
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout:Device: 5h/5d	Inode: 910         Links: 1
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout:Access: 2024-05-17 14:49:19.598743875 +0000
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout:Modify: 2024-05-17 14:49:19.358746019 +0000
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout:Change: 2024-05-17 14:49:19.358746019 +0000
2024-05-17T14:54:52.502 INFO:teuthology.orchestra.run.smithi072.stdout: Birth: -
2024-05-17T14:54:52.502 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1
2024-05-17T14:54:52.556 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records in
2024-05-17T14:54:52.556 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records out
2024-05-17T14:54:52.556 INFO:teuthology.orchestra.run.smithi072.stderr:512 bytes copied, 0.000410933 s, 1.2 MB/s
2024-05-17T14:54:52.558 DEBUG:teuthology.orchestra.run.smithi072:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3
2024-05-17T14:54:52.608 DEBUG:teuthology.orchestra.run.smithi072:> stat /dev/vg_nvme/lv_4
2024-05-17T14:54:52.653 INFO:teuthology.orchestra.run.smithi072.stdout:  File: /dev/vg_nvme/lv_4 -> ../dm-3
2024-05-17T14:54:52.653 INFO:teuthology.orchestra.run.smithi072.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:54:52.653 INFO:teuthology.orchestra.run.smithi072.stdout:Device: 5h/5d	Inode: 926         Links: 1
2024-05-17T14:54:52.653 INFO:teuthology.orchestra.run.smithi072.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:54:52.653 INFO:teuthology.orchestra.run.smithi072.stdout:Access: 2024-05-17 14:49:20.090739481 +0000
2024-05-17T14:54:52.653 INFO:teuthology.orchestra.run.smithi072.stdout:Modify: 2024-05-17 14:49:19.842741696 +0000
2024-05-17T14:54:52.654 INFO:teuthology.orchestra.run.smithi072.stdout:Change: 2024-05-17 14:49:19.842741696 +0000
2024-05-17T14:54:52.654 INFO:teuthology.orchestra.run.smithi072.stdout: Birth: -
2024-05-17T14:54:52.654 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1
2024-05-17T14:54:52.707 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records in
2024-05-17T14:54:52.708 INFO:teuthology.orchestra.run.smithi072.stderr:1+0 records out
2024-05-17T14:54:52.708 INFO:teuthology.orchestra.run.smithi072.stderr:512 bytes copied, 0.000382763 s, 1.3 MB/s
2024-05-17T14:54:52.709 DEBUG:teuthology.orchestra.run.smithi072:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4
2024-05-17T14:54:52.758 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:54:52.758 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi072.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}}
2024-05-17T14:54:52.758 INFO:tasks.ceph:Generating config...
2024-05-17T14:54:52.759 INFO:tasks.ceph:[client] debug rbd = 20
2024-05-17T14:54:52.759 INFO:tasks.ceph:[client] debug rbd_mirror = 15
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client] log to stderr = False
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client] rbd default features = 125
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client] rbd mirror image policy type = none
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:52.760 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:52.761 INFO:tasks.ceph:[global] mon client directed command retry = 5
2024-05-17T14:54:52.761 INFO:tasks.ceph:[global] mon warn on pool no app = False
2024-05-17T14:54:52.761 INFO:tasks.ceph:[global] ms inject socket failures = 5000
2024-05-17T14:54:52.761 INFO:tasks.ceph:[mgr] debug mgr = 20
2024-05-17T14:54:52.761 INFO:tasks.ceph:[mgr] debug ms = 1
2024-05-17T14:54:52.761 INFO:tasks.ceph:[mon] debug mon = 20
2024-05-17T14:54:52.761 INFO:tasks.ceph:[mon] debug ms = 1
2024-05-17T14:54:52.761 INFO:tasks.ceph:[mon] debug paxos = 20
2024-05-17T14:54:52.761 INFO:tasks.ceph:[osd] bdev async discard = True
2024-05-17T14:54:52.761 INFO:tasks.ceph:[osd] bdev enable discard = True
2024-05-17T14:54:52.761 INFO:tasks.ceph:[osd] bluestore allocator = stupid
2024-05-17T14:54:52.761 INFO:tasks.ceph:[osd] bluestore block size = 96636764160
2024-05-17T14:54:52.761 INFO:tasks.ceph:[osd] bluestore fsck on mount = True
2024-05-17T14:54:52.761 INFO:tasks.ceph:[osd] debug bluefs = 1/20
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] debug bluestore = 1/20
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] debug ms = 1
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] debug osd = 20
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] debug rocksdb = 4/10
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95
2024-05-17T14:54:52.762 INFO:tasks.ceph:[osd] osd objectstore = bluestore
2024-05-17T14:54:52.762 INFO:tasks.ceph:Setting up cluster1.mon.a...
2024-05-17T14:54:52.763 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring
2024-05-17T14:54:52.832 INFO:teuthology.orchestra.run.smithi072.stdout:creating /etc/ceph/cluster1.keyring
2024-05-17T14:54:52.850 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring
2024-05-17T14:54:52.927 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod 0644 /etc/ceph/cluster1.keyring
2024-05-17T14:54:52.979 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '172.21.15.72')]
2024-05-17T14:54:52.980 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.72', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}}
2024-05-17T14:54:52.981 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:52.981 DEBUG:teuthology.orchestra.run.smithi072:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf
2024-05-17T14:54:53.025 DEBUG:teuthology.orchestra.run.smithi072:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.72 --print /home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:54:53.102 INFO:teuthology.orchestra.run.smithi072.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:monmaptool: generated fsid 719cd724-687e-4be9-8a5d-75127dd56f0a
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:setting min_mon_release = quincy
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:epoch 0
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:fsid 719cd724-687e-4be9-8a5d-75127dd56f0a
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:last_changed 2024-05-17T14:54:53.106070+0000
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:created 2024-05-17T14:54:53.106070+0000
2024-05-17T14:54:53.103 INFO:teuthology.orchestra.run.smithi072.stdout:min_mon_release 17 (quincy)
2024-05-17T14:54:53.104 INFO:teuthology.orchestra.run.smithi072.stdout:election_strategy: 1
2024-05-17T14:54:53.104 INFO:teuthology.orchestra.run.smithi072.stdout:0: [v2:172.21.15.72:3300/0,v1:172.21.15.72:6789/0] mon.a
2024-05-17T14:54:53.104 INFO:teuthology.orchestra.run.smithi072.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors)
2024-05-17T14:54:53.105 DEBUG:teuthology.orchestra.run.smithi072:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf
2024-05-17T14:54:53.155 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 719cd724-687e-4be9-8a5d-75127dd56f0a...
2024-05-17T14:54:53.156 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null
2024-05-17T14:54:53.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null
2024-05-17T14:54:53.225 INFO:teuthology.orchestra.run.smithi072.stdout:[global]
2024-05-17T14:54:53.225 INFO:teuthology.orchestra.run.smithi072.stdout:	chdir = ""
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/$cluster-$name.pid
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	auth supported = cephx
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	filestore xattr use omap = true
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	mon clock drift allowed = 1.000
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	osd crush chooseleaf type = 0
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	auth debug = true
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	ms die on old message = true
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:	ms die on bug = true
2024-05-17T14:54:53.226 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon max pg per osd = 10000	# >= luminous
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon pg warn max object skew = 0
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	# disable pg_autoscaler by default for new pools
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	osd_pool_default_pg_autoscale_mode = off
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	osd pool default size = 2
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd allow primary affinity = true
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd allow pg remap = true
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on legacy crush tunables = false
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on crush straw calc version zero = false
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on no sortbitwise = false
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on osd down out interval zero = false
2024-05-17T14:54:53.227 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on too few osds = false
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_pool_pg_num_not_power_of_two = false
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_pool_no_redundancy = false
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_allow_pool_size_one = true
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	osd default data pool replay window = 5
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon allow pool delete = true
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon cluster log file level = debug
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	debug asserts on shutdown = true
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon health detail to clog = false
2024-05-17T14:54:53.228 INFO:teuthology.orchestra.run.smithi072.stdout:	mon host = 172.21.15.72
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	mon client directed command retry = 5
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on pool no app = False
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	ms inject socket failures = 5000
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	fsid = 719cd724-687e-4be9-8a5d-75127dd56f0a
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:[osd]
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	osd journal size = 100
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	osd scrub load threshold = 5.0
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	osd scrub max interval = 600
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	osd mclock profile = high_recovery_ops
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	osd recover clone overlap = true
2024-05-17T14:54:53.229 INFO:teuthology.orchestra.run.smithi072.stdout:	osd recovery max chunk = 1048576
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug shutdown = true
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug op order = true
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug verify stray on activate = true
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug trim objects = true
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd open classes on start = true
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug pg log writeout = true
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd deep scrub update digest min age = 30
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	osd map max advance = 10
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.230 INFO:teuthology.orchestra.run.smithi072.stdout:	journal zero on create = true
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	filestore ondisk finisher threads = 3
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	filestore apply finisher threads = 3
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	bdev debug aio = true
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug misdirected ops = true
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	bdev async discard = True
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	bdev enable discard = True
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	bluestore allocator = stupid
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	bluestore block size = 96636764160
2024-05-17T14:54:53.231 INFO:teuthology.orchestra.run.smithi072.stdout:	bluestore fsck on mount = True
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	debug bluefs = 1/20
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	debug bluestore = 1/20
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	debug ms = 1
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	debug osd = 20
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	debug rocksdb = 4/10
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd backfillfull_ratio = 0.85
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd full ratio = 0.9
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd nearfull ratio = 0.8
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	osd failsafe full ratio = 0.95
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	osd objectstore = bluestore
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:[mgr]
2024-05-17T14:54:53.232 INFO:teuthology.orchestra.run.smithi072.stdout:	debug ms = 1
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug mgr = 20
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug mon = 20
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug auth = 20
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	mgr/telemetry/nag = false
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:[mon]
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug ms = 1
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug mon = 20
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug paxos = 20
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	debug auth = 20
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	mon data avail warn = 5
2024-05-17T14:54:53.233 INFO:teuthology.orchestra.run.smithi072.stdout:	mon mgr mkfs grace = 240
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd reporter subtree level = osd
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd prime pg temp = true
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	# rotate auth tickets quickly to exercise renewal paths
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	auth mon ticket ttl = 660	# 11m
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	auth service ticket ttl = 240	# 4m
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	# don't complain about insecure global_id in the test suite
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_insecure_global_id_reclaim = false
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_insecure_global_id_reclaim_allowed = false
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.234 INFO:teuthology.orchestra.run.smithi072.stdout:	# 1m isn't quite enough
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_down_mkfs_grace = 2m
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_filestore_osds = false
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:[client]
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	rgw cache enabled = true
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	rgw enable ops log = true
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	rgw enable usage log = true
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	log file = /var/log/ceph/$cluster-$name.$pid.log
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/$cluster-$name.$pid.asok
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	debug rbd = 20
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	debug rbd_mirror = 15
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	log to stderr = False
2024-05-17T14:54:53.235 INFO:teuthology.orchestra.run.smithi072.stdout:	rbd default features = 125
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	rbd mirror image policy type = none
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:[mon.a]
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.0]
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.1]
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.2]
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.3]
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.236 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.237 INFO:teuthology.orchestra.run.smithi171.stdout:[global]
2024-05-17T14:54:53.237 INFO:teuthology.orchestra.run.smithi171.stdout:	chdir = ""
2024-05-17T14:54:53.237 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/$cluster-$name.pid
2024-05-17T14:54:53.237 INFO:teuthology.orchestra.run.smithi171.stdout:	auth supported = cephx
2024-05-17T14:54:53.237 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.237 INFO:teuthology.orchestra.run.smithi171.stdout:	filestore xattr use omap = true
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	mon clock drift allowed = 1.000
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	osd crush chooseleaf type = 0
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	auth debug = true
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	ms die on old message = true
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	ms die on bug = true
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	mon max pg per osd = 10000	# >= luminous
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	mon pg warn max object skew = 0
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.238 INFO:teuthology.orchestra.run.smithi171.stdout:	# disable pg_autoscaler by default for new pools
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	osd_pool_default_pg_autoscale_mode = off
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	osd pool default size = 2
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd allow primary affinity = true
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd allow pg remap = true
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on legacy crush tunables = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on crush straw calc version zero = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on no sortbitwise = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on osd down out interval zero = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on too few osds = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_pool_pg_num_not_power_of_two = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_pool_no_redundancy = false
2024-05-17T14:54:53.239 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_allow_pool_size_one = true
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	osd default data pool replay window = 5
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	mon allow pool delete = true
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	mon cluster log file level = debug
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	debug asserts on shutdown = true
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	mon health detail to clog = false
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	mon host = 172.21.15.72
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	mon client directed command retry = 5
2024-05-17T14:54:53.240 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on pool no app = False
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	ms inject socket failures = 5000
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	fsid = 719cd724-687e-4be9-8a5d-75127dd56f0a
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:[osd]
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	osd journal size = 100
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	osd scrub load threshold = 5.0
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	osd scrub max interval = 600
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	osd mclock profile = high_recovery_ops
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	osd recover clone overlap = true
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:	osd recovery max chunk = 1048576
2024-05-17T14:54:53.241 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug shutdown = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug op order = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug verify stray on activate = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug trim objects = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd open classes on start = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug pg log writeout = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd deep scrub update digest min age = 30
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	osd map max advance = 10
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:	journal zero on create = true
2024-05-17T14:54:53.242 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	filestore ondisk finisher threads = 3
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	filestore apply finisher threads = 3
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	bdev debug aio = true
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug misdirected ops = true
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	bdev async discard = True
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	bdev enable discard = True
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	bluestore allocator = stupid
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	bluestore block size = 96636764160
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	bluestore fsck on mount = True
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	debug bluefs = 1/20
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	debug bluestore = 1/20
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	debug ms = 1
2024-05-17T14:54:53.243 INFO:teuthology.orchestra.run.smithi171.stdout:	debug osd = 20
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	debug rocksdb = 4/10
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd backfillfull_ratio = 0.85
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd full ratio = 0.9
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd nearfull ratio = 0.8
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	osd failsafe full ratio = 0.95
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	osd objectstore = bluestore
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:[mgr]
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	debug ms = 1
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	debug mgr = 20
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	debug mon = 20
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	debug auth = 20
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:54:53.244 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mgr/telemetry/nag = false
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:[mon]
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	debug ms = 1
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	debug mon = 20
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	debug paxos = 20
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	debug auth = 20
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mon data avail warn = 5
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mon mgr mkfs grace = 240
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd reporter subtree level = osd
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd prime pg temp = true
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.245 INFO:teuthology.orchestra.run.smithi171.stdout:	# rotate auth tickets quickly to exercise renewal paths
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	auth mon ticket ttl = 660	# 11m
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	auth service ticket ttl = 240	# 4m
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	# don't complain about insecure global_id in the test suite
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_insecure_global_id_reclaim = false
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_insecure_global_id_reclaim_allowed = false
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	# 1m isn't quite enough
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_down_mkfs_grace = 2m
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_filestore_osds = false
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:[client]
2024-05-17T14:54:53.246 INFO:teuthology.orchestra.run.smithi171.stdout:	rgw cache enabled = true
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	rgw enable ops log = true
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	rgw enable usage log = true
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	log file = /var/log/ceph/$cluster-$name.$pid.log
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/$cluster-$name.$pid.asok
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	debug rbd = 20
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	debug rbd_mirror = 15
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	log to stderr = False
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	rbd default features = 125
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	rbd mirror image policy type = none
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:[mon.a]
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.0]
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.247 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.1]
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.2]
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.3]
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:54:53.248 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:54:53.249 INFO:tasks.ceph:Creating admin key on cluster1.mon.a...
2024-05-17T14:54:53.250 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring
2024-05-17T14:54:53.340 INFO:tasks.ceph:Copying monmap to all nodes...
2024-05-17T14:54:53.341 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:53.341 DEBUG:teuthology.orchestra.run.smithi072:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout
2024-05-17T14:54:53.347 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:53.348 DEBUG:teuthology.orchestra.run.smithi072:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout
2024-05-17T14:54:53.393 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi072.front.sepia.ceph.com
2024-05-17T14:54:53.393 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:53.393 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/etc/ceph/cluster1.keyring
2024-05-17T14:54:53.394 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod 0644 /etc/ceph/cluster1.keyring
2024-05-17T14:54:53.455 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:54:53.455 DEBUG:teuthology.orchestra.run.smithi072:> dd of=/home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:54:53.501 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:54:53.501 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:54:53.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/cluster1.keyring
2024-05-17T14:54:53.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 0644 /etc/ceph/cluster1.keyring
2024-05-17T14:54:53.524 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:54:53.524 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:54:53.572 INFO:tasks.ceph:Setting up mon nodes...
2024-05-17T14:54:53.573 INFO:tasks.ceph:Setting up mgr nodes...
2024-05-17T14:54:53.573 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring
2024-05-17T14:54:53.613 INFO:teuthology.orchestra.run.smithi072.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring
2024-05-17T14:54:53.615 INFO:tasks.ceph:Setting up mds nodes...
2024-05-17T14:54:53.616 INFO:tasks.ceph_client:Setting up client nodes...
2024-05-17T14:54:53.616 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring
2024-05-17T14:54:53.689 INFO:teuthology.orchestra.run.smithi072.stdout:creating /etc/ceph/cluster1.client.0.keyring
2024-05-17T14:54:53.699 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring
2024-05-17T14:54:53.737 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.keyring
2024-05-17T14:54:53.769 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring
2024-05-17T14:54:53.850 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring
2024-05-17T14:54:53.861 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring
2024-05-17T14:54:53.935 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring
2024-05-17T14:54:53.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring
2024-05-17T14:54:54.017 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring
2024-05-17T14:54:54.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring
2024-05-17T14:54:54.104 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring
2024-05-17T14:54:54.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring
2024-05-17T14:54:54.184 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring
2024-05-17T14:54:54.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring
2024-05-17T14:54:54.264 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring
2024-05-17T14:54:54.273 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring
2024-05-17T14:54:54.344 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring
2024-05-17T14:54:54.355 INFO:tasks.ceph:Running mkfs on osd nodes...
2024-05-17T14:54:54.355 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi072.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}}
2024-05-17T14:54:54.355 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0
2024-05-17T14:54:54.370 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:54:54.370 INFO:tasks.ceph:role: cluster1.osd.0
2024-05-17T14:54:54.370 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi072.front.sepia.ceph.com
2024-05-17T14:54:54.370 DEBUG:teuthology.orchestra.run.smithi072:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1
2024-05-17T14:54:54.432 INFO:teuthology.orchestra.run.smithi072.stdout:meta-data=/dev/vg_nvme/lv_1      isize=2048   agcount=4, agsize=5859072 blks
2024-05-17T14:54:54.432 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sectsz=512   attr=2, projid32bit=1
2024-05-17T14:54:54.432 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       crc=1        finobt=1, sparse=1, rmapbt=0
2024-05-17T14:54:54.432 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       reflink=1    bigtime=0 inobtcount=0
2024-05-17T14:54:54.432 INFO:teuthology.orchestra.run.smithi072.stdout:data     =                       bsize=4096   blocks=23436288, imaxpct=25
2024-05-17T14:54:54.433 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sunit=0      swidth=0 blks
2024-05-17T14:54:54.433 INFO:teuthology.orchestra.run.smithi072.stdout:naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
2024-05-17T14:54:54.433 INFO:teuthology.orchestra.run.smithi072.stdout:log      =internal log           bsize=4096   blocks=11443, version=2
2024-05-17T14:54:54.433 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sectsz=512   sunit=0 blks, lazy-count=1
2024-05-17T14:54:54.433 INFO:teuthology.orchestra.run.smithi072.stdout:realtime =none                   extsz=4096   blocks=0, rtextents=0
2024-05-17T14:54:54.911 INFO:teuthology.orchestra.run.smithi072.stdout:Discarding blocks...Done.
2024-05-17T14:54:54.913 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi072.front.sepia.ceph.com -o noatime
2024-05-17T14:54:54.913 DEBUG:teuthology.orchestra.run.smithi072:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0
2024-05-17T14:54:55.042 DEBUG:teuthology.orchestra.run.smithi072:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0
2024-05-17T14:54:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:54:55.054 INFO:teuthology.orchestra.run.smithi072.stderr:sudo: /sbin/restorecon: command not found
2024-05-17T14:54:55.054 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1
2024-05-17T14:54:55.108 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:54:55.108 INFO:tasks.ceph:role: cluster1.osd.1
2024-05-17T14:54:55.109 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi072.front.sepia.ceph.com
2024-05-17T14:54:55.109 DEBUG:teuthology.orchestra.run.smithi072:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2
2024-05-17T14:54:55.172 INFO:teuthology.orchestra.run.smithi072.stdout:meta-data=/dev/vg_nvme/lv_2      isize=2048   agcount=4, agsize=5859072 blks
2024-05-17T14:54:55.172 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sectsz=512   attr=2, projid32bit=1
2024-05-17T14:54:55.172 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       crc=1        finobt=1, sparse=1, rmapbt=0
2024-05-17T14:54:55.172 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       reflink=1    bigtime=0 inobtcount=0
2024-05-17T14:54:55.172 INFO:teuthology.orchestra.run.smithi072.stdout:data     =                       bsize=4096   blocks=23436288, imaxpct=25
2024-05-17T14:54:55.172 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sunit=0      swidth=0 blks
2024-05-17T14:54:55.173 INFO:teuthology.orchestra.run.smithi072.stdout:naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
2024-05-17T14:54:55.173 INFO:teuthology.orchestra.run.smithi072.stdout:log      =internal log           bsize=4096   blocks=11443, version=2
2024-05-17T14:54:55.173 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sectsz=512   sunit=0 blks, lazy-count=1
2024-05-17T14:54:55.173 INFO:teuthology.orchestra.run.smithi072.stdout:realtime =none                   extsz=4096   blocks=0, rtextents=0
2024-05-17T14:54:55.601 INFO:teuthology.orchestra.run.smithi072.stdout:Discarding blocks...Done.
2024-05-17T14:54:55.603 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi072.front.sepia.ceph.com -o noatime
2024-05-17T14:54:55.603 DEBUG:teuthology.orchestra.run.smithi072:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1
2024-05-17T14:54:55.665 DEBUG:teuthology.orchestra.run.smithi072:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1
2024-05-17T14:54:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:54:55.717 INFO:teuthology.orchestra.run.smithi072.stderr:sudo: /sbin/restorecon: command not found
2024-05-17T14:54:55.718 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2
2024-05-17T14:54:55.770 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:54:55.770 INFO:tasks.ceph:role: cluster1.osd.2
2024-05-17T14:54:55.771 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi072.front.sepia.ceph.com
2024-05-17T14:54:55.771 DEBUG:teuthology.orchestra.run.smithi072:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:meta-data=/dev/vg_nvme/lv_3      isize=2048   agcount=4, agsize=5859072 blks
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sectsz=512   attr=2, projid32bit=1
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       crc=1        finobt=1, sparse=1, rmapbt=0
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       reflink=1    bigtime=0 inobtcount=0
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:data     =                       bsize=4096   blocks=23436288, imaxpct=25
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sunit=0      swidth=0 blks
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
2024-05-17T14:54:55.831 INFO:teuthology.orchestra.run.smithi072.stdout:log      =internal log           bsize=4096   blocks=11443, version=2
2024-05-17T14:54:55.832 INFO:teuthology.orchestra.run.smithi072.stdout:         =                       sectsz=512   sunit=0 blks, lazy-count=1
2024-05-17T14:54:55.832 INFO:teuthology.orchestra.run.smithi072.stdout:realtime =none                   extsz=4096   blocks=0, rtextents=0
2024-05-17T14:54:56.289 INFO:teuthology.orchestra.run.smithi072.stdout:Discarding blocks...Done.
2024-05-17T14:54:56.291 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi072.front.sepia.ceph.com -o noatime
2024-05-17T14:54:56.291 DEBUG:teuthology.orchestra.run.smithi072:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2
2024-05-17T14:54:56.385 DEBUG:teuthology.orchestra.run.smithi072:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2
2024-05-17T14:54:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:54:56.395 INFO:teuthology.orchestra.run.smithi072.stderr:sudo: /sbin/restorecon: command not found
2024-05-17T14:54:56.396 DEBUG:teuthology.orchestra.run.smithi072:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:54:56.510 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:56.510+0000 7f45177b17c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory
2024-05-17T14:54:56.513 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:56.514+0000 7f45177b17c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring
2024-05-17T14:54:56.513 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:56.514+0000 7f45177b17c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-0/block: (2) No such file or directory
2024-05-17T14:54:56.513 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:56.514+0000 7f45177b17c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-0/block: (2) No such file or directory
2024-05-17T14:54:56.513 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:56.514+0000 7f45177b17c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-0/block: (2) No such file or directory
2024-05-17T14:54:56.518 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:56.518+0000 7f45177b17c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid
2024-05-17T14:54:59.503 DEBUG:teuthology.orchestra.run.smithi072:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0
2024-05-17T14:54:59.516 DEBUG:teuthology.orchestra.run.smithi072:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:54:59.598 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:59.598+0000 7f73b58b57c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory
2024-05-17T14:54:59.599 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:59.598+0000 7f73b58b57c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring
2024-05-17T14:54:59.599 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:59.598+0000 7f73b58b57c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-1/block: (2) No such file or directory
2024-05-17T14:54:59.599 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:59.598+0000 7f73b58b57c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-1/block: (2) No such file or directory
2024-05-17T14:54:59.599 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:59.598+0000 7f73b58b57c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-1/block: (2) No such file or directory
2024-05-17T14:54:59.599 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:54:59.598+0000 7f73b58b57c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid
2024-05-17T14:55:02.576 DEBUG:teuthology.orchestra.run.smithi072:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1
2024-05-17T14:55:02.632 DEBUG:teuthology.orchestra.run.smithi072:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:55:02.713 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:55:02.714+0000 7f150ef7a7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory
2024-05-17T14:55:02.714 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:55:02.714+0000 7f150ef7a7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring
2024-05-17T14:55:02.714 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:55:02.714+0000 7f150ef7a7c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-2/block: (2) No such file or directory
2024-05-17T14:55:02.714 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:55:02.714+0000 7f150ef7a7c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-2/block: (2) No such file or directory
2024-05-17T14:55:02.714 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:55:02.714+0000 7f150ef7a7c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster1-2/block: (2) No such file or directory
2024-05-17T14:55:02.715 INFO:teuthology.orchestra.run.smithi072.stderr:2024-05-17T14:55:02.714+0000 7f150ef7a7c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid
2024-05-17T14:55:05.686 DEBUG:teuthology.orchestra.run.smithi072:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2
2024-05-17T14:55:05.741 INFO:tasks.ceph:Reading keys from all nodes...
2024-05-17T14:55:05.741 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:05.741 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout
2024-05-17T14:55:05.796 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:05.796 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout
2024-05-17T14:55:05.851 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:05.852 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout
2024-05-17T14:55:05.905 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:05.905 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout
2024-05-17T14:55:05.961 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:05.961 DEBUG:teuthology.orchestra.run.smithi072:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout
2024-05-17T14:55:06.009 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.009 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout
2024-05-17T14:55:06.015 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.015 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout
2024-05-17T14:55:06.060 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.061 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout
2024-05-17T14:55:06.109 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.109 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout
2024-05-17T14:55:06.156 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.157 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout
2024-05-17T14:55:06.204 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.204 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout
2024-05-17T14:55:06.252 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.252 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout
2024-05-17T14:55:06.300 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:06.300 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout
2024-05-17T14:55:06.348 INFO:tasks.ceph:Adding keys to all mons...
2024-05-17T14:55:06.349 DEBUG:teuthology.orchestra.run.smithi072:> sudo tee -a /etc/ceph/cluster1.keyring
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:[mgr.x]
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC9b0dmr+3AJBAAiG+aHTZdcPINgVPx0bMHBw==
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:[osd.0]
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQDAb0dm3D2dHhAAq2DU4LRSjfhf4QfUbo15mg==
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:[osd.1]
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQDDb0dm52DXIxAAJGDdQx4O7KrZap5QkZEDKw==
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:[osd.2]
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQDGb0dmoBy+KhAAVDvS3JtX/PPnKe4vcFENnA==
2024-05-17T14:55:06.361 INFO:teuthology.orchestra.run.smithi072.stdout:[client.0]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC9b0dmu3VDKRAANcXaB3Ky/TyQEfa9fAC/mw==
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC9b0dms0kFLBAAoLxftY/5mb3ZXKGRzf9FqA==
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.0]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC9b0dmfpi+MhAAJ9RfgV78UVEO18VyDKH0aQ==
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.1]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC9b0dmJh7TNxAAyBv11LYHGiv0t1WZkf/fGw==
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.2]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC+b0dmSoMgARAAv8eHMZjUUiY3xOf81CQwlg==
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.3]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC+b0dmtqtFBhAAb7TIIXQIE4gxIVPMMvFMJg==
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.4]
2024-05-17T14:55:06.362 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC+b0dmNYIKCxAAAyUqSfkZ0jc+ZaxdAxAugw==
2024-05-17T14:55:06.363 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.5]
2024-05-17T14:55:06.363 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC+b0dm55zRDxAA+CeRz32/vxupXOu0XnrZgQ==
2024-05-17T14:55:06.363 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.6]
2024-05-17T14:55:06.363 INFO:teuthology.orchestra.run.smithi072.stdout:	key = AQC+b0dmu0efFBAAhyyTQpyzDYj5fS+zkh1aWQ==
2024-05-17T14:55:06.364 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *'
2024-05-17T14:55:06.453 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
2024-05-17T14:55:06.495 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
2024-05-17T14:55:06.580 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
2024-05-17T14:55:06.671 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:06.759 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:06.801 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:06.887 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:06.975 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:07.064 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:07.106 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:07.193 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:07.235 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:07.324 INFO:tasks.ceph:Running mkfs on mon nodes...
2024-05-17T14:55:07.324 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a
2024-05-17T14:55:07.336 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring
2024-05-17T14:55:07.428 DEBUG:teuthology.orchestra.run.smithi072:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a
2024-05-17T14:55:07.483 DEBUG:teuthology.orchestra.run.smithi072:> rm -- /home/ubuntu/cephtest/cluster1.monmap
2024-05-17T14:55:07.538 INFO:tasks.ceph:Starting mon daemons in cluster cluster1...
2024-05-17T14:55:07.539 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon
2024-05-17T14:55:07.539 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a
2024-05-17T14:55:07.583 INFO:tasks.ceph.cluster1.mon.a:Started
2024-05-17T14:55:07.584 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1...
2024-05-17T14:55:07.584 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon
2024-05-17T14:55:07.584 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x
2024-05-17T14:55:07.588 INFO:tasks.ceph.cluster1.mgr.x:Started
2024-05-17T14:55:07.588 INFO:tasks.ceph:Setting crush tunables to default
2024-05-17T14:55:07.589 DEBUG:teuthology.orchestra.run.smithi072:> sudo ceph --cluster cluster1 osd crush tunables default
2024-05-17T14:55:07.815 INFO:teuthology.orchestra.run.smithi072.stderr:adjusted tunables profile to default
2024-05-17T14:55:07.834 INFO:tasks.ceph:check_enable_crimson: False
2024-05-17T14:55:07.834 INFO:tasks.ceph:Starting osd daemons in cluster cluster1...
2024-05-17T14:55:07.835 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:07.835 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout
2024-05-17T14:55:07.848 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:07.848 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout
2024-05-17T14:55:07.903 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:07.903 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout
2024-05-17T14:55:07.942 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:07.942+0000 7f86168f4180 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member
2024-05-17T14:55:07.953 DEBUG:teuthology.orchestra.run.smithi072:> sudo ceph --cluster cluster1 osd new bc344a61-9540-42bb-83ae-cdc339f0b34f 0
2024-05-17T14:55:08.200 INFO:teuthology.orchestra.run.smithi072.stdout:0
2024-05-17T14:55:08.216 DEBUG:teuthology.orchestra.run.smithi072:> sudo ceph --cluster cluster1 osd new 230545b6-da54-44a9-850f-6b625c60ef52 1
2024-05-17T14:55:08.407 INFO:teuthology.orchestra.run.smithi072.stdout:1
2024-05-17T14:55:08.424 DEBUG:teuthology.orchestra.run.smithi072:> sudo ceph --cluster cluster1 osd new 834d34c8-86ef-4d9f-95fa-077e9b893762 2
2024-05-17T14:55:08.490 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:08.490+0000 7f86168f4180 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member
2024-05-17T14:55:08.618 INFO:teuthology.orchestra.run.smithi072.stdout:2
2024-05-17T14:55:08.634 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon
2024-05-17T14:55:08.634 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0
2024-05-17T14:55:08.636 INFO:tasks.ceph.cluster1.osd.0:Started
2024-05-17T14:55:08.636 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon
2024-05-17T14:55:08.636 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1
2024-05-17T14:55:08.638 INFO:tasks.ceph.cluster1.osd.1:Started
2024-05-17T14:55:08.638 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon
2024-05-17T14:55:08.638 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2
2024-05-17T14:55:08.640 INFO:tasks.ceph.cluster1.osd.2:Started
2024-05-17T14:55:08.640 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json
2024-05-17T14:55:08.710 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:08.710+0000 7f86168f4180 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member
2024-05-17T14:55:08.820 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:08.820 INFO:teuthology.orchestra.run.smithi072.stdout:{"epoch":5,"fsid":"719cd724-687e-4be9-8a5d-75127dd56f0a","created":"2024-05-17T14:55:07.717289+0000","modified":"2024-05-17T14:55:08.620734+0000","last_up_change":"0.000000","last_in_change":"2024-05-17T14:55:08.620734+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"bc344a61-9540-42bb-83ae-cdc339f0b34f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"230545b6-da54-44a9-850f-6b625c60ef52","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"834d34c8-86ef-4d9f-95fa-077e9b893762","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:08.836 INFO:tasks.ceph.ceph_manager.cluster1:[]
2024-05-17T14:55:08.836 INFO:tasks.ceph:Waiting for OSDs to come up
2024-05-17T14:55:09.141 DEBUG:teuthology.orchestra.run.smithi072:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json
2024-05-17T14:55:09.258 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T14:55:09.258+0000 7fd08bb2f7c0 -1 Falling back to public interface
2024-05-17T14:55:09.261 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T14:55:09.262+0000 7f076d1f57c0 -1 Falling back to public interface
2024-05-17T14:55:09.261 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T14:55:09.262+0000 7fc16dc4f7c0 -1 Falling back to public interface
2024-05-17T14:55:09.266 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.266+0000 7f86168f4180 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member
2024-05-17T14:55:09.322 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.322+0000 7f86168f4180 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member
2024-05-17T14:55:09.330 INFO:teuthology.misc.health.smithi072.stdout:
2024-05-17T14:55:09.331 INFO:teuthology.misc.health.smithi072.stdout:{"epoch":5,"fsid":"719cd724-687e-4be9-8a5d-75127dd56f0a","created":"2024-05-17T14:55:07.717289+0000","modified":"2024-05-17T14:55:08.620734+0000","last_up_change":"0.000000","last_in_change":"2024-05-17T14:55:08.620734+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"bc344a61-9540-42bb-83ae-cdc339f0b34f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"230545b6-da54-44a9-850f-6b625c60ef52","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"834d34c8-86ef-4d9f-95fa-077e9b893762","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:09.348 DEBUG:teuthology.misc:0 of 3 OSDs are up
2024-05-17T14:55:09.376 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.378+0000 7f86168f4180 -1 mgr[py] Module progress has missing NOTIFY_TYPES member
2024-05-17T14:55:09.453 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.454+0000 7f86168f4180 -1 mgr[py] Module crash has missing NOTIFY_TYPES member
2024-05-17T14:55:09.504 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.506+0000 7f86168f4180 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member
2024-05-17T14:55:09.581 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.582+0000 7f86168f4180 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member
2024-05-17T14:55:09.634 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.634+0000 7f86168f4180 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member
2024-05-17T14:55:09.783 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.782+0000 7f86168f4180 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member
2024-05-17T14:55:09.844 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:09.846+0000 7f86168f4180 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member
2024-05-17T14:55:10.884 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode.
2024-05-17T14:55:10.884 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve.
2024-05-17T14:55:10.884 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:  from numpy import show_config as show_numpy_config
2024-05-17T14:55:10.969 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:10.970+0000 7f86168f4180 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member
2024-05-17T14:55:11.152 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:11.150+0000 7f86168f4180 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member
2024-05-17T14:55:11.328 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:11.330+0000 7f86168f4180 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member
2024-05-17T14:55:11.397 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:11.398+0000 7f86168f4180 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member
2024-05-17T14:55:11.531 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:11.530+0000 7f86168f4180 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member
2024-05-17T14:55:12.190 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:12.190+0000 7f86168f4180 -1 mgr[py] Module influx has missing NOTIFY_TYPES member
2024-05-17T14:55:12.271 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:12.270+0000 7f86168f4180 -1 mgr[py] Module status has missing NOTIFY_TYPES member
2024-05-17T14:55:12.413 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T14:55:12.414+0000 7fc16dc4f7c0 -1 osd.0 0 log_to_monitors true
2024-05-17T14:55:12.501 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:12.502+0000 7f86168f4180 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member
2024-05-17T14:55:12.575 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:12.574+0000 7f86168f4180 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member
2024-05-17T14:55:12.865 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:12.866+0000 7f86168f4180 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member
2024-05-17T14:55:12.927 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:12.926+0000 7f86168f4180 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member
2024-05-17T14:55:13.080 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T14:55:13.082+0000 7f076d1f57c0 -1 osd.1 0 log_to_monitors true
2024-05-17T14:55:13.101 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T14:55:13.102+0000 7fd08bb2f7c0 -1 osd.2 0 log_to_monitors true
2024-05-17T14:55:13.962 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T14:55:13.962+0000 7f85bd538640 -1 mgr.server handle_report got status from non-daemon mon.a
2024-05-17T14:55:14.473 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T14:55:14.474+0000 7fc169bf8640 -1 osd.0 0 waiting for initial osdmap
2024-05-17T14:55:14.531 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T14:55:14.530+0000 7fc161a2c640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory
2024-05-17T14:55:15.651 DEBUG:teuthology.orchestra.run.smithi072:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json
2024-05-17T14:55:16.032 INFO:teuthology.misc.health.smithi072.stdout:
2024-05-17T14:55:16.032 INFO:teuthology.misc.health.smithi072.stdout:{"epoch":9,"fsid":"719cd724-687e-4be9-8a5d-75127dd56f0a","created":"2024-05-17T14:55:07.717289+0000","modified":"2024-05-17T14:55:15.482412+0000","last_up_change":"2024-05-17T14:55:15.482412+0000","last_in_change":"2024-05-17T14:55:08.620734+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"bc344a61-9540-42bb-83ae-cdc339f0b34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6809","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6811","nonce":3906540648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6813","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6815","nonce":3906540648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6821","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6823","nonce":3906540648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6817","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6819","nonce":3906540648}]},"public_addr":"172.21.15.72:6811/3906540648","cluster_addr":"172.21.15.72:6815/3906540648","heartbeat_back_addr":"172.21.15.72:6823/3906540648","heartbeat_front_addr":"172.21.15.72:6819/3906540648","state":["exists","up"]},{"osd":1,"uuid":"230545b6-da54-44a9-850f-6b625c60ef52","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"834d34c8-86ef-4d9f-95fa-077e9b893762","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:16.056 DEBUG:teuthology.misc:1 of 3 OSDs are up
2024-05-17T14:55:16.066 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T14:55:16.066+0000 7f076919e640 -1 osd.1 0 waiting for initial osdmap
2024-05-17T14:55:16.096 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T14:55:16.094+0000 7fd0882ea640 -1 osd.2 0 waiting for initial osdmap
2024-05-17T14:55:16.101 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T14:55:16.102+0000 7f0760fd2640 -1 osd.1 9 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory
2024-05-17T14:55:16.125 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T14:55:16.126+0000 7fd07f90c640 -1 osd.2 9 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory
2024-05-17T14:55:22.360 DEBUG:teuthology.orchestra.run.smithi072:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json
2024-05-17T14:55:22.655 INFO:teuthology.misc.health.smithi072.stdout:
2024-05-17T14:55:22.655 INFO:teuthology.misc.health.smithi072.stdout:{"epoch":14,"fsid":"719cd724-687e-4be9-8a5d-75127dd56f0a","created":"2024-05-17T14:55:07.717289+0000","modified":"2024-05-17T14:55:21.506377+0000","last_up_change":"2024-05-17T14:55:17.100707+0000","last_in_change":"2024-05-17T14:55:08.620734+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-17T14:55:19.012336+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc344a61-9540-42bb-83ae-cdc339f0b34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6809","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6811","nonce":3906540648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6813","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6815","nonce":3906540648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6821","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6823","nonce":3906540648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6817","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6819","nonce":3906540648}]},"public_addr":"172.21.15.72:6811/3906540648","cluster_addr":"172.21.15.72:6815/3906540648","heartbeat_back_addr":"172.21.15.72:6823/3906540648","heartbeat_front_addr":"172.21.15.72:6819/3906540648","state":["exists","up"]},{"osd":1,"uuid":"230545b6-da54-44a9-850f-6b625c60ef52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6808","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6810","nonce":3312931660}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6812","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6814","nonce":3312931660}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6820","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6822","nonce":3312931660}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6816","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6818","nonce":3312931660}]},"public_addr":"172.21.15.72:6810/3312931660","cluster_addr":"172.21.15.72:6814/3312931660","heartbeat_back_addr":"172.21.15.72:6822/3312931660","heartbeat_front_addr":"172.21.15.72:6818/3312931660","state":["exists","up"]},{"osd":2,"uuid":"834d34c8-86ef-4d9f-95fa-077e9b893762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6800","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6801","nonce":2868769902}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6802","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6803","nonce":2868769902}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6806","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6807","nonce":2868769902}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6804","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6805","nonce":2868769902}]},"public_addr":"172.21.15.72:6801/2868769902","cluster_addr":"172.21.15.72:6803/2868769902","heartbeat_back_addr":"172.21.15.72:6807/2868769902","heartbeat_front_addr":"172.21.15.72:6805/2868769902","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:13.451551+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:14.073465+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:14.106946+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:22.674 DEBUG:teuthology.misc:3 of 3 OSDs are up
2024-05-17T14:55:22.675 INFO:tasks.ceph:Creating RBD pool
2024-05-17T14:55:22.675 DEBUG:teuthology.orchestra.run.smithi072:> sudo ceph --cluster cluster1 osd pool create rbd 8
2024-05-17T14:55:23.515 INFO:teuthology.orchestra.run.smithi072.stderr:pool 'rbd' created
2024-05-17T14:55:23.543 DEBUG:teuthology.orchestra.run.smithi072:> sudo ceph --cluster cluster1 osd pool application enable rbd rbd --yes-i-really-mean-it
2024-05-17T14:55:24.517 INFO:teuthology.orchestra.run.smithi072.stderr:enabled application 'rbd' on pool 'rbd'
2024-05-17T14:55:24.547 INFO:tasks.ceph:Starting mds daemons in cluster cluster1...
2024-05-17T14:55:24.548 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean...
2024-05-17T14:55:24.548 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available
2024-05-17T14:55:24.548 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json
2024-05-17T14:55:24.548 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting
2024-05-17T14:55:24.883 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:24.899 INFO:teuthology.orchestra.run.smithi072.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6824","nonce":840559846},{"type":"v1","addr":"172.21.15.72:6825","nonce":840559846}]},"active_addr":"172.21.15.72:6825/840559846","active_change":"2024-05-17T14:55:12.949223+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace.  Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB.  Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":2843475133}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":1878610798}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":895175178}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":2016714538}]}]}
2024-05-17T14:55:24.903 INFO:tasks.ceph.ceph_manager.cluster1:mgr available!
2024-05-17T14:55:24.903 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up
2024-05-17T14:55:24.903 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json
2024-05-17T14:55:25.199 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:25.200 INFO:teuthology.orchestra.run.smithi072.stdout:{"epoch":16,"fsid":"719cd724-687e-4be9-8a5d-75127dd56f0a","created":"2024-05-17T14:55:07.717289+0000","modified":"2024-05-17T14:55:24.518830+0000","last_up_change":"2024-05-17T14:55:17.100707+0000","last_in_change":"2024-05-17T14:55:08.620734+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-17T14:55:19.012336+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-05-17T14:55:22.973346+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc344a61-9540-42bb-83ae-cdc339f0b34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6809","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6811","nonce":3906540648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6813","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6815","nonce":3906540648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6821","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6823","nonce":3906540648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6817","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6819","nonce":3906540648}]},"public_addr":"172.21.15.72:6811/3906540648","cluster_addr":"172.21.15.72:6815/3906540648","heartbeat_back_addr":"172.21.15.72:6823/3906540648","heartbeat_front_addr":"172.21.15.72:6819/3906540648","state":["exists","up"]},{"osd":1,"uuid":"230545b6-da54-44a9-850f-6b625c60ef52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6808","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6810","nonce":3312931660}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6812","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6814","nonce":3312931660}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6820","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6822","nonce":3312931660}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6816","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6818","nonce":3312931660}]},"public_addr":"172.21.15.72:6810/3312931660","cluster_addr":"172.21.15.72:6814/3312931660","heartbeat_back_addr":"172.21.15.72:6822/3312931660","heartbeat_front_addr":"172.21.15.72:6818/3312931660","state":["exists","up"]},{"osd":2,"uuid":"834d34c8-86ef-4d9f-95fa-077e9b893762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6800","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6801","nonce":2868769902}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6802","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6803","nonce":2868769902}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6806","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6807","nonce":2868769902}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6804","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6805","nonce":2868769902}]},"public_addr":"172.21.15.72:6801/2868769902","cluster_addr":"172.21.15.72:6803/2868769902","heartbeat_back_addr":"172.21.15.72:6807/2868769902","heartbeat_front_addr":"172.21.15.72:6805/2868769902","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:13.451551+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:14.073465+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:14.106946+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:25.218 INFO:tasks.ceph.ceph_manager.cluster1:all up!
2024-05-17T14:55:25.218 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json
2024-05-17T14:55:25.506 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:25.506 INFO:teuthology.orchestra.run.smithi072.stdout:{"epoch":16,"fsid":"719cd724-687e-4be9-8a5d-75127dd56f0a","created":"2024-05-17T14:55:07.717289+0000","modified":"2024-05-17T14:55:24.518830+0000","last_up_change":"2024-05-17T14:55:17.100707+0000","last_in_change":"2024-05-17T14:55:08.620734+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-17T14:55:19.012336+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-05-17T14:55:22.973346+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc344a61-9540-42bb-83ae-cdc339f0b34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6809","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6811","nonce":3906540648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6813","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6815","nonce":3906540648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6821","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6823","nonce":3906540648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6817","nonce":3906540648},{"type":"v1","addr":"172.21.15.72:6819","nonce":3906540648}]},"public_addr":"172.21.15.72:6811/3906540648","cluster_addr":"172.21.15.72:6815/3906540648","heartbeat_back_addr":"172.21.15.72:6823/3906540648","heartbeat_front_addr":"172.21.15.72:6819/3906540648","state":["exists","up"]},{"osd":1,"uuid":"230545b6-da54-44a9-850f-6b625c60ef52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6808","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6810","nonce":3312931660}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6812","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6814","nonce":3312931660}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6820","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6822","nonce":3312931660}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6816","nonce":3312931660},{"type":"v1","addr":"172.21.15.72:6818","nonce":3312931660}]},"public_addr":"172.21.15.72:6810/3312931660","cluster_addr":"172.21.15.72:6814/3312931660","heartbeat_back_addr":"172.21.15.72:6822/3312931660","heartbeat_front_addr":"172.21.15.72:6818/3312931660","state":["exists","up"]},{"osd":2,"uuid":"834d34c8-86ef-4d9f-95fa-077e9b893762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6800","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6801","nonce":2868769902}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6802","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6803","nonce":2868769902}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6806","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6807","nonce":2868769902}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6804","nonce":2868769902},{"type":"v1","addr":"172.21.15.72:6805","nonce":2868769902}]},"public_addr":"172.21.15.72:6801/2868769902","cluster_addr":"172.21.15.72:6803/2868769902","heartbeat_back_addr":"172.21.15.72:6807/2868769902","heartbeat_front_addr":"172.21.15.72:6805/2868769902","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:13.451551+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:14.073465+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:14.106946+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:25.525 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats
2024-05-17T14:55:25.525 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats
2024-05-17T14:55:25.526 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats
2024-05-17T14:55:25.683 INFO:teuthology.orchestra.run.smithi072.stdout:38654705668
2024-05-17T14:55:25.683 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0
2024-05-17T14:55:25.684 INFO:teuthology.orchestra.run.smithi072.stdout:42949672965
2024-05-17T14:55:25.685 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1
2024-05-17T14:55:25.687 INFO:teuthology.orchestra.run.smithi072.stdout:42949672965
2024-05-17T14:55:25.687 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2
2024-05-17T14:55:25.976 INFO:teuthology.orchestra.run.smithi072.stdout:38654705667
2024-05-17T14:55:25.994 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.0
2024-05-17T14:55:26.010 INFO:teuthology.orchestra.run.smithi072.stdout:42949672964
2024-05-17T14:55:26.021 INFO:teuthology.orchestra.run.smithi072.stdout:42949672964
2024-05-17T14:55:26.027 INFO:tasks.ceph.ceph_manager.cluster1:need seq 42949672965 got 42949672964 for osd.1
2024-05-17T14:55:26.037 INFO:tasks.ceph.ceph_manager.cluster1:need seq 42949672965 got 42949672964 for osd.2
2024-05-17T14:55:26.995 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0
2024-05-17T14:55:27.028 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1
2024-05-17T14:55:27.038 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2
2024-05-17T14:55:27.298 INFO:teuthology.orchestra.run.smithi072.stdout:38654705667
2024-05-17T14:55:27.315 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.0
2024-05-17T14:55:27.324 INFO:teuthology.orchestra.run.smithi072.stdout:42949672964
2024-05-17T14:55:27.329 INFO:teuthology.orchestra.run.smithi072.stdout:42949672964
2024-05-17T14:55:27.342 INFO:tasks.ceph.ceph_manager.cluster1:need seq 42949672965 got 42949672964 for osd.2
2024-05-17T14:55:27.347 INFO:tasks.ceph.ceph_manager.cluster1:need seq 42949672965 got 42949672964 for osd.1
2024-05-17T14:55:28.317 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0
2024-05-17T14:55:28.343 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2
2024-05-17T14:55:28.348 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1
2024-05-17T14:55:28.653 INFO:teuthology.orchestra.run.smithi072.stdout:42949672965
2024-05-17T14:55:28.669 INFO:tasks.ceph.ceph_manager.cluster1:need seq 42949672965 got 42949672965 for osd.1
2024-05-17T14:55:28.670 DEBUG:teuthology.parallel:result is None
2024-05-17T14:55:28.676 INFO:teuthology.orchestra.run.smithi072.stdout:38654705669
2024-05-17T14:55:28.693 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705669 for osd.0
2024-05-17T14:55:28.693 DEBUG:teuthology.parallel:result is None
2024-05-17T14:55:28.693 INFO:teuthology.orchestra.run.smithi072.stdout:42949672965
2024-05-17T14:55:28.711 INFO:tasks.ceph.ceph_manager.cluster1:need seq 42949672965 got 42949672965 for osd.2
2024-05-17T14:55:28.711 DEBUG:teuthology.parallel:result is None
2024-05-17T14:55:28.711 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean
2024-05-17T14:55:28.711 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json
2024-05-17T14:55:29.021 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:29.021 INFO:teuthology.orchestra.run.smithi072.stderr:dumped all
2024-05-17T14:55:29.037 INFO:teuthology.orchestra.run.smithi072.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-05-17T14:55:28.960731+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":4,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82648,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032872,"statfs":{"total":289910292480,"available":289825660928,"internally_reserved":0,"allocated":1622016,"data_stored":1300675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4789,"internal_metadata":82963787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001333"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.009042+0000","last_change":"2024-05-17T14:55:24.531853+0000","last_active":"2024-05-17T14:55:26.009042+0000","last_peered":"2024-05-17T14:55:26.009042+0000","last_clean":"2024-05-17T14:55:26.009042+0000","last_became_active":"2024-05-17T14:55:24.531451+0000","last_became_peered":"2024-05-17T14:55:24.531451+0000","last_unstale":"2024-05-17T14:55:26.009042+0000","last_undegraded":"2024-05-17T14:55:26.009042+0000","last_fullsized":"2024-05-17T14:55:26.009042+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T16:17:36.852360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008937+0000","last_change":"2024-05-17T14:55:24.532861+0000","last_active":"2024-05-17T14:55:26.008937+0000","last_peered":"2024-05-17T14:55:26.008937+0000","last_clean":"2024-05-17T14:55:26.008937+0000","last_became_active":"2024-05-17T14:55:24.532443+0000","last_became_peered":"2024-05-17T14:55:24.532443+0000","last_unstale":"2024-05-17T14:55:26.008937+0000","last_undegraded":"2024-05-17T14:55:26.008937+0000","last_fullsized":"2024-05-17T14:55:26.008937+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T01:37:10.500808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008784+0000","last_change":"2024-05-17T14:55:24.532761+0000","last_active":"2024-05-17T14:55:26.008784+0000","last_peered":"2024-05-17T14:55:26.008784+0000","last_clean":"2024-05-17T14:55:26.008784+0000","last_became_active":"2024-05-17T14:55:24.532371+0000","last_became_peered":"2024-05-17T14:55:24.532371+0000","last_unstale":"2024-05-17T14:55:26.008784+0000","last_undegraded":"2024-05-17T14:55:26.008784+0000","last_fullsized":"2024-05-17T14:55:26.008784+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:35:38.586253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.009022+0000","last_change":"2024-05-17T14:55:24.531720+0000","last_active":"2024-05-17T14:55:26.009022+0000","last_peered":"2024-05-17T14:55:26.009022+0000","last_clean":"2024-05-17T14:55:26.009022+0000","last_became_active":"2024-05-17T14:55:24.531358+0000","last_became_peered":"2024-05-17T14:55:24.531358+0000","last_unstale":"2024-05-17T14:55:26.009022+0000","last_undegraded":"2024-05-17T14:55:26.009022+0000","last_fullsized":"2024-05-17T14:55:26.009022+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T00:13:09.108295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-05-17T14:55:24.539760+0000","last_change":"2024-05-17T14:55:24.534770+0000","last_active":"2024-05-17T14:55:24.539760+0000","last_peered":"2024-05-17T14:55:24.539760+0000","last_clean":"2024-05-17T14:55:24.539760+0000","last_became_active":"2024-05-17T14:55:24.533999+0000","last_became_peered":"2024-05-17T14:55:24.533999+0000","last_unstale":"2024-05-17T14:55:24.539760+0000","last_undegraded":"2024-05-17T14:55:24.539760+0000","last_fullsized":"2024-05-17T14:55:24.539760+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T18:52:08.923899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:25.530477+0000","last_change":"2024-05-17T14:55:24.533599+0000","last_active":"2024-05-17T14:55:25.530477+0000","last_peered":"2024-05-17T14:55:25.530477+0000","last_clean":"2024-05-17T14:55:25.530477+0000","last_became_active":"2024-05-17T14:55:24.533039+0000","last_became_peered":"2024-05-17T14:55:24.533039+0000","last_unstale":"2024-05-17T14:55:25.530477+0000","last_undegraded":"2024-05-17T14:55:25.530477+0000","last_fullsized":"2024-05-17T14:55:25.530477+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T19:42:10.008809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:25.530478+0000","last_change":"2024-05-17T14:55:24.533454+0000","last_active":"2024-05-17T14:55:25.530478+0000","last_peered":"2024-05-17T14:55:25.530478+0000","last_clean":"2024-05-17T14:55:25.530478+0000","last_became_active":"2024-05-17T14:55:24.533085+0000","last_became_peered":"2024-05-17T14:55:24.533085+0000","last_unstale":"2024-05-17T14:55:25.530478+0000","last_undegraded":"2024-05-17T14:55:25.530478+0000","last_fullsized":"2024-05-17T14:55:25.530478+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:05:05.901060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008898+0000","last_change":"2024-05-17T14:55:24.531206+0000","last_active":"2024-05-17T14:55:26.008898+0000","last_peered":"2024-05-17T14:55:26.008898+0000","last_clean":"2024-05-17T14:55:26.008898+0000","last_became_active":"2024-05-17T14:55:24.530751+0000","last_became_peered":"2024-05-17T14:55:24.530751+0000","last_unstale":"2024-05-17T14:55:26.008898+0000","last_undegraded":"2024-05-17T14:55:26.008898+0000","last_fullsized":"2024-05-17T14:55:26.008898+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:23:29.623097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"13'52","reported_seq":85,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008740+0000","last_change":"2024-05-17T14:55:21.126927+0000","last_active":"2024-05-17T14:55:26.008740+0000","last_peered":"2024-05-17T14:55:26.008740+0000","last_clean":"2024-05-17T14:55:26.008740+0000","last_became_active":"2024-05-17T14:55:21.124409+0000","last_became_peered":"2024-05-17T14:55:21.124409+0000","last_unstale":"2024-05-17T14:55:26.008740+0000","last_undegraded":"2024-05-17T14:55:26.008740+0000","last_fullsized":"2024-05-17T14:55:26.008740+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:19.495325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:19.495325+0000","last_clean_scrub_stamp":"2024-05-17T14:55:19.495325+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:57:09.590106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":10,"seq":42949672966,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":147456,"data_stored":41139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":10,"seq":42949672966,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":741376,"data_stored":631507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27736,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344104,"statfs":{"total":96636764160,"available":96608362496,"internally_reserved":0,"allocated":733184,"data_stored":628029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
2024-05-17T14:55:29.042 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json
2024-05-17T14:55:29.306 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:29.306 INFO:teuthology.orchestra.run.smithi072.stderr:dumped all
2024-05-17T14:55:29.322 INFO:teuthology.orchestra.run.smithi072.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-05-17T14:55:28.960731+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":4,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82648,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032872,"statfs":{"total":289910292480,"available":289825660928,"internally_reserved":0,"allocated":1622016,"data_stored":1300675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4789,"internal_metadata":82963787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001333"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.009042+0000","last_change":"2024-05-17T14:55:24.531853+0000","last_active":"2024-05-17T14:55:26.009042+0000","last_peered":"2024-05-17T14:55:26.009042+0000","last_clean":"2024-05-17T14:55:26.009042+0000","last_became_active":"2024-05-17T14:55:24.531451+0000","last_became_peered":"2024-05-17T14:55:24.531451+0000","last_unstale":"2024-05-17T14:55:26.009042+0000","last_undegraded":"2024-05-17T14:55:26.009042+0000","last_fullsized":"2024-05-17T14:55:26.009042+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T16:17:36.852360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008937+0000","last_change":"2024-05-17T14:55:24.532861+0000","last_active":"2024-05-17T14:55:26.008937+0000","last_peered":"2024-05-17T14:55:26.008937+0000","last_clean":"2024-05-17T14:55:26.008937+0000","last_became_active":"2024-05-17T14:55:24.532443+0000","last_became_peered":"2024-05-17T14:55:24.532443+0000","last_unstale":"2024-05-17T14:55:26.008937+0000","last_undegraded":"2024-05-17T14:55:26.008937+0000","last_fullsized":"2024-05-17T14:55:26.008937+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T01:37:10.500808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008784+0000","last_change":"2024-05-17T14:55:24.532761+0000","last_active":"2024-05-17T14:55:26.008784+0000","last_peered":"2024-05-17T14:55:26.008784+0000","last_clean":"2024-05-17T14:55:26.008784+0000","last_became_active":"2024-05-17T14:55:24.532371+0000","last_became_peered":"2024-05-17T14:55:24.532371+0000","last_unstale":"2024-05-17T14:55:26.008784+0000","last_undegraded":"2024-05-17T14:55:26.008784+0000","last_fullsized":"2024-05-17T14:55:26.008784+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:35:38.586253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.009022+0000","last_change":"2024-05-17T14:55:24.531720+0000","last_active":"2024-05-17T14:55:26.009022+0000","last_peered":"2024-05-17T14:55:26.009022+0000","last_clean":"2024-05-17T14:55:26.009022+0000","last_became_active":"2024-05-17T14:55:24.531358+0000","last_became_peered":"2024-05-17T14:55:24.531358+0000","last_unstale":"2024-05-17T14:55:26.009022+0000","last_undegraded":"2024-05-17T14:55:26.009022+0000","last_fullsized":"2024-05-17T14:55:26.009022+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T00:13:09.108295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-05-17T14:55:24.539760+0000","last_change":"2024-05-17T14:55:24.534770+0000","last_active":"2024-05-17T14:55:24.539760+0000","last_peered":"2024-05-17T14:55:24.539760+0000","last_clean":"2024-05-17T14:55:24.539760+0000","last_became_active":"2024-05-17T14:55:24.533999+0000","last_became_peered":"2024-05-17T14:55:24.533999+0000","last_unstale":"2024-05-17T14:55:24.539760+0000","last_undegraded":"2024-05-17T14:55:24.539760+0000","last_fullsized":"2024-05-17T14:55:24.539760+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T18:52:08.923899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:25.530477+0000","last_change":"2024-05-17T14:55:24.533599+0000","last_active":"2024-05-17T14:55:25.530477+0000","last_peered":"2024-05-17T14:55:25.530477+0000","last_clean":"2024-05-17T14:55:25.530477+0000","last_became_active":"2024-05-17T14:55:24.533039+0000","last_became_peered":"2024-05-17T14:55:24.533039+0000","last_unstale":"2024-05-17T14:55:25.530477+0000","last_undegraded":"2024-05-17T14:55:25.530477+0000","last_fullsized":"2024-05-17T14:55:25.530477+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T19:42:10.008809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:25.530478+0000","last_change":"2024-05-17T14:55:24.533454+0000","last_active":"2024-05-17T14:55:25.530478+0000","last_peered":"2024-05-17T14:55:25.530478+0000","last_clean":"2024-05-17T14:55:25.530478+0000","last_became_active":"2024-05-17T14:55:24.533085+0000","last_became_peered":"2024-05-17T14:55:24.533085+0000","last_unstale":"2024-05-17T14:55:25.530478+0000","last_undegraded":"2024-05-17T14:55:25.530478+0000","last_fullsized":"2024-05-17T14:55:25.530478+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:05:05.901060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008898+0000","last_change":"2024-05-17T14:55:24.531206+0000","last_active":"2024-05-17T14:55:26.008898+0000","last_peered":"2024-05-17T14:55:26.008898+0000","last_clean":"2024-05-17T14:55:26.008898+0000","last_became_active":"2024-05-17T14:55:24.530751+0000","last_became_peered":"2024-05-17T14:55:24.530751+0000","last_unstale":"2024-05-17T14:55:26.008898+0000","last_undegraded":"2024-05-17T14:55:26.008898+0000","last_fullsized":"2024-05-17T14:55:26.008898+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:23.512470+0000","last_clean_scrub_stamp":"2024-05-17T14:55:23.512470+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:23:29.623097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"13'52","reported_seq":85,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-05-17T14:55:26.008740+0000","last_change":"2024-05-17T14:55:21.126927+0000","last_active":"2024-05-17T14:55:26.008740+0000","last_peered":"2024-05-17T14:55:26.008740+0000","last_clean":"2024-05-17T14:55:26.008740+0000","last_became_active":"2024-05-17T14:55:21.124409+0000","last_became_peered":"2024-05-17T14:55:21.124409+0000","last_unstale":"2024-05-17T14:55:26.008740+0000","last_undegraded":"2024-05-17T14:55:26.008740+0000","last_fullsized":"2024-05-17T14:55:26.008740+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:19.495325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:19.495325+0000","last_clean_scrub_stamp":"2024-05-17T14:55:19.495325+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T17:57:09.590106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":10,"seq":42949672966,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":147456,"data_stored":41139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":10,"seq":42949672966,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":741376,"data_stored":631507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27736,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344104,"statfs":{"total":96636764160,"available":96608362496,"internally_reserved":0,"allocated":733184,"data_stored":628029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
2024-05-17T14:55:29.323 INFO:tasks.ceph.ceph_manager.cluster1:clean!
2024-05-17T14:55:29.323 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy...
2024-05-17T14:55:29.323 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy
2024-05-17T14:55:29.323 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json
2024-05-17T14:55:29.632 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:29.632 INFO:teuthology.orchestra.run.smithi072.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]}
2024-05-17T14:55:29.649 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done
2024-05-17T14:55:29.649 INFO:teuthology.run_tasks:Running task ceph...
2024-05-17T14:55:29.659 INFO:tasks.ceph:Creating ceph cluster cluster2...
2024-05-17T14:55:29.659 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True}
2024-05-17T14:55:29.659 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi/7711217', 'branch': 'wip-dis-testing', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'email': 'idryomov@gmail.com', 'first_in_suite': False, 'job_id': '7711217', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'dis-2024-05-17_14:29:27-rbd-wip-dis-testing-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-dis-testing'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-dis-testing', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, 'owner': 'scheduled_dis@teuthology', 'priority': 70, 'repo': 'https://github.com/ceph/ceph-ci', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 3754, 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4', 'sleep_before_teardown': 0, 'subset': '56/99', 'suite': 'rbd', 'suite_branch': 'wip-dis-testing', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci', 'suite_sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4', 'targets': {'smithi072.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAtFxZM3PX3EqFG5Suxok8lQhwmW0wU9dQiMCx105AEPlaLR+Mmd/eylLsthBupi9TbcCp1qT5241HblZVZk0AQ=', 'smithi171.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG6ZXSZlCBgx+DF91uuAf3IuG4mnOVQsHYBVdotBdA7HEH0Zygg0Zvz4SN+EXfOn2SaSabD5rx9Bw8uZG/mFa8k='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'ee59d86aad04089481e55c1a15aa5871339935b4'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'wip-fix-extra-system-packages', 'teuthology_sha1': 'cad7edd8d899a7c59a48f7da3294fca4cb778f64', 'timestamp': '2024-05-17_14:29:27', 'tube': 'smithi', 'user': 'dis', 'verbose': True, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885'}
2024-05-17T14:55:29.660 DEBUG:teuthology.orchestra.run.smithi072:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data
2024-05-17T14:55:29.662 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data
2024-05-17T14:55:29.668 DEBUG:teuthology.orchestra.run.smithi072:> sudo install -d -m0777 -- /var/run/ceph
2024-05-17T14:55:29.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo install -d -m0777 -- /var/run/ceph
2024-05-17T14:55:29.724 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:29.724 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/scratch_devs of=/dev/stdout
2024-05-17T14:55:29.776 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4']
2024-05-17T14:55:29.776 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_1
2024-05-17T14:55:29.824 INFO:teuthology.orchestra.run.smithi171.stdout:  File: /dev/vg_nvme/lv_1 -> ../dm-0
2024-05-17T14:55:29.824 INFO:teuthology.orchestra.run.smithi171.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:55:29.824 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d	Inode: 874         Links: 1
2024-05-17T14:55:29.825 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:55:29.825 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-05-17 14:49:20.215920548 +0000
2024-05-17T14:55:29.825 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-05-17 14:49:19.967922763 +0000
2024-05-17T14:55:29.825 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-05-17 14:49:19.967922763 +0000
2024-05-17T14:55:29.825 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: -
2024-05-17T14:55:29.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1
2024-05-17T14:55:29.879 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in
2024-05-17T14:55:29.879 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out
2024-05-17T14:55:29.879 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000450103 s, 1.1 MB/s
2024-05-17T14:55:29.880 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1
2024-05-17T14:55:29.929 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_2
2024-05-17T14:55:29.976 INFO:teuthology.orchestra.run.smithi171.stdout:  File: /dev/vg_nvme/lv_2 -> ../dm-1
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d	Inode: 888         Links: 1
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-05-17 14:49:20.691916297 +0000
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-05-17 14:49:20.467918297 +0000
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-05-17 14:49:20.467918297 +0000
2024-05-17T14:55:29.977 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: -
2024-05-17T14:55:29.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1
2024-05-17T14:55:30.029 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in
2024-05-17T14:55:30.029 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out
2024-05-17T14:55:30.029 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000304426 s, 1.7 MB/s
2024-05-17T14:55:30.031 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2
2024-05-17T14:55:30.077 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_3
2024-05-17T14:55:30.124 INFO:teuthology.orchestra.run.smithi171.stdout:  File: /dev/vg_nvme/lv_3 -> ../dm-2
2024-05-17T14:55:30.124 INFO:teuthology.orchestra.run.smithi171.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:55:30.124 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d	Inode: 905         Links: 1
2024-05-17T14:55:30.125 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:55:30.125 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-05-17 14:49:20.919914260 +0000
2024-05-17T14:55:30.125 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-05-17 14:49:20.915914296 +0000
2024-05-17T14:55:30.125 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-05-17 14:49:20.915914296 +0000
2024-05-17T14:55:30.125 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: -
2024-05-17T14:55:30.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1
2024-05-17T14:55:30.179 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in
2024-05-17T14:55:30.179 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out
2024-05-17T14:55:30.179 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.00030234 s, 1.7 MB/s
2024-05-17T14:55:30.180 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3
2024-05-17T14:55:30.229 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_4
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:  File: /dev/vg_nvme/lv_4 -> ../dm-3
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:  Size: 7         	Blocks: 0          IO Block: 4096   symbolic link
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d	Inode: 928         Links: 1
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-05-17 14:49:21.679907471 +0000
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-05-17 14:49:21.411909866 +0000
2024-05-17T14:55:30.276 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-05-17 14:49:21.411909866 +0000
2024-05-17T14:55:30.277 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: -
2024-05-17T14:55:30.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1
2024-05-17T14:55:30.330 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in
2024-05-17T14:55:30.330 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out
2024-05-17T14:55:30.330 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000387893 s, 1.3 MB/s
2024-05-17T14:55:30.331 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4
2024-05-17T14:55:30.381 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:55:30.381 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi171.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}}
2024-05-17T14:55:30.381 INFO:tasks.ceph:Generating config...
2024-05-17T14:55:30.382 INFO:tasks.ceph:[client] debug rbd = 20
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client] debug rbd_mirror = 15
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client] log to stderr = False
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client] rbd default features = 125
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client] rbd mirror image policy type = none
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.383 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.384 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.384 INFO:tasks.ceph:[global] mon client directed command retry = 5
2024-05-17T14:55:30.384 INFO:tasks.ceph:[global] mon warn on pool no app = False
2024-05-17T14:55:30.384 INFO:tasks.ceph:[global] ms inject socket failures = 5000
2024-05-17T14:55:30.384 INFO:tasks.ceph:[mgr] debug mgr = 20
2024-05-17T14:55:30.384 INFO:tasks.ceph:[mgr] debug ms = 1
2024-05-17T14:55:30.384 INFO:tasks.ceph:[mon] debug mon = 20
2024-05-17T14:55:30.384 INFO:tasks.ceph:[mon] debug ms = 1
2024-05-17T14:55:30.384 INFO:tasks.ceph:[mon] debug paxos = 20
2024-05-17T14:55:30.384 INFO:tasks.ceph:[osd] bdev async discard = True
2024-05-17T14:55:30.384 INFO:tasks.ceph:[osd] bdev enable discard = True
2024-05-17T14:55:30.384 INFO:tasks.ceph:[osd] bluestore allocator = stupid
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] bluestore block size = 96636764160
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] bluestore fsck on mount = True
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] debug bluefs = 1/20
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] debug bluestore = 1/20
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] debug ms = 1
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] debug osd = 20
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] debug rocksdb = 4/10
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95
2024-05-17T14:55:30.385 INFO:tasks.ceph:[osd] osd objectstore = bluestore
2024-05-17T14:55:30.386 INFO:tasks.ceph:Setting up cluster2.mon.a...
2024-05-17T14:55:30.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring
2024-05-17T14:55:30.454 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.keyring
2024-05-17T14:55:30.457 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring
2024-05-17T14:55:30.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 0644 /etc/ceph/cluster2.keyring
2024-05-17T14:55:30.591 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '172.21.15.171')]
2024-05-17T14:55:30.592 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.171', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}}
2024-05-17T14:55:30.593 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:30.593 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf
2024-05-17T14:55:30.640 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.171 --print /home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:monmaptool: generated fsid 81895d15-a91d-4358-a42f-b54a4e77bb53
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:setting min_mon_release = quincy
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:epoch 0
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:fsid 81895d15-a91d-4358-a42f-b54a4e77bb53
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:last_changed 2024-05-17T14:55:30.719417+0000
2024-05-17T14:55:30.718 INFO:teuthology.orchestra.run.smithi171.stdout:created 2024-05-17T14:55:30.719417+0000
2024-05-17T14:55:30.719 INFO:teuthology.orchestra.run.smithi171.stdout:min_mon_release 17 (quincy)
2024-05-17T14:55:30.719 INFO:teuthology.orchestra.run.smithi171.stdout:election_strategy: 1
2024-05-17T14:55:30.719 INFO:teuthology.orchestra.run.smithi171.stdout:0: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.a
2024-05-17T14:55:30.719 INFO:teuthology.orchestra.run.smithi171.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors)
2024-05-17T14:55:30.720 DEBUG:teuthology.orchestra.run.smithi171:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf
2024-05-17T14:55:30.768 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 81895d15-a91d-4358-a42f-b54a4e77bb53...
2024-05-17T14:55:30.769 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null
2024-05-17T14:55:30.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null
2024-05-17T14:55:30.814 INFO:teuthology.orchestra.run.smithi072.stdout:[global]
2024-05-17T14:55:30.814 INFO:teuthology.orchestra.run.smithi072.stdout:	chdir = ""
2024-05-17T14:55:30.814 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/$cluster-$name.pid
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	auth supported = cephx
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	filestore xattr use omap = true
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	mon clock drift allowed = 1.000
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	osd crush chooseleaf type = 0
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	auth debug = true
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	ms die on old message = true
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	ms die on bug = true
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	mon max pg per osd = 10000	# >= luminous
2024-05-17T14:55:30.815 INFO:teuthology.orchestra.run.smithi072.stdout:	mon pg warn max object skew = 0
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	# disable pg_autoscaler by default for new pools
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	osd_pool_default_pg_autoscale_mode = off
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	osd pool default size = 2
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd allow primary affinity = true
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd allow pg remap = true
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on legacy crush tunables = false
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on crush straw calc version zero = false
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on no sortbitwise = false
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on osd down out interval zero = false
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on too few osds = false
2024-05-17T14:55:30.816 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_pool_pg_num_not_power_of_two = false
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_pool_no_redundancy = false
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_allow_pool_size_one = true
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	osd default data pool replay window = 5
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon allow pool delete = true
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon cluster log file level = debug
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	debug asserts on shutdown = true
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon health detail to clog = false
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon host = 172.21.15.171
2024-05-17T14:55:30.817 INFO:teuthology.orchestra.run.smithi072.stdout:	mon client directed command retry = 5
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	mon warn on pool no app = False
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	ms inject socket failures = 5000
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	fsid = 81895d15-a91d-4358-a42f-b54a4e77bb53
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:[osd]
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	osd journal size = 100
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	osd scrub load threshold = 5.0
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	osd scrub max interval = 600
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	osd mclock profile = high_recovery_ops
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	osd recover clone overlap = true
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:	osd recovery max chunk = 1048576
2024-05-17T14:55:30.818 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug shutdown = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug op order = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug verify stray on activate = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug trim objects = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd open classes on start = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug pg log writeout = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd deep scrub update digest min age = 30
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	osd map max advance = 10
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	journal zero on create = true
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.819 INFO:teuthology.orchestra.run.smithi072.stdout:	filestore ondisk finisher threads = 3
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	filestore apply finisher threads = 3
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	bdev debug aio = true
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	osd debug misdirected ops = true
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	bdev async discard = True
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	bdev enable discard = True
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	bluestore allocator = stupid
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	bluestore block size = 96636764160
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	bluestore fsck on mount = True
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	debug bluefs = 1/20
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	debug bluestore = 1/20
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	debug ms = 1
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	debug osd = 20
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	debug rocksdb = 4/10
2024-05-17T14:55:30.820 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd backfillfull_ratio = 0.85
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd full ratio = 0.9
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd nearfull ratio = 0.8
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	osd failsafe full ratio = 0.95
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	osd objectstore = bluestore
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:[mgr]
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	debug ms = 1
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	debug mgr = 20
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	debug mon = 20
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	debug auth = 20
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:	mgr/telemetry/nag = false
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.821 INFO:teuthology.orchestra.run.smithi072.stdout:[mon]
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	debug ms = 1
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	debug mon = 20
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	debug paxos = 20
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	debug auth = 20
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	mon data avail warn = 5
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	mon mgr mkfs grace = 240
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd reporter subtree level = osd
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	mon osd prime pg temp = true
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	# rotate auth tickets quickly to exercise renewal paths
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	auth mon ticket ttl = 660	# 11m
2024-05-17T14:55:30.822 INFO:teuthology.orchestra.run.smithi072.stdout:	auth service ticket ttl = 240	# 4m
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	# don't complain about insecure global_id in the test suite
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_insecure_global_id_reclaim = false
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_insecure_global_id_reclaim_allowed = false
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	# 1m isn't quite enough
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_down_mkfs_grace = 2m
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	mon_warn_on_filestore_osds = false
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:[client]
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	rgw cache enabled = true
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	rgw enable ops log = true
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	rgw enable usage log = true
2024-05-17T14:55:30.823 INFO:teuthology.orchestra.run.smithi072.stdout:	log file = /var/log/ceph/$cluster-$name.$pid.log
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/$cluster-$name.$pid.asok
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	debug rbd = 20
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	debug rbd_mirror = 15
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	log to stderr = False
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	rbd default features = 125
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	rbd mirror image policy type = none
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:[mon.a]
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.0]
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.1]
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.824 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.2]
2024-05-17T14:55:30.825 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.825 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.825 INFO:teuthology.orchestra.run.smithi072.stdout:[client.mirror.3]
2024-05-17T14:55:30.825 INFO:teuthology.orchestra.run.smithi072.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.825 INFO:teuthology.orchestra.run.smithi072.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:[global]
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	chdir = ""
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/$cluster-$name.pid
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	auth supported = cephx
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	filestore xattr use omap = true
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	mon clock drift allowed = 1.000
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	osd crush chooseleaf type = 0
2024-05-17T14:55:30.841 INFO:teuthology.orchestra.run.smithi171.stdout:	auth debug = true
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	ms die on old message = true
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	ms die on bug = true
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	mon max pg per osd = 10000	# >= luminous
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	mon pg warn max object skew = 0
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	# disable pg_autoscaler by default for new pools
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	osd_pool_default_pg_autoscale_mode = off
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	osd pool default size = 2
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.842 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd allow primary affinity = true
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd allow pg remap = true
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on legacy crush tunables = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on crush straw calc version zero = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on no sortbitwise = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on osd down out interval zero = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on too few osds = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_pool_pg_num_not_power_of_two = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_pool_no_redundancy = false
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_allow_pool_size_one = true
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:	osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
2024-05-17T14:55:30.843 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	osd default data pool replay window = 5
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	mon allow pool delete = true
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	mon cluster log file level = debug
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	debug asserts on shutdown = true
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	mon health detail to clog = false
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	mon host = 172.21.15.171
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	mon client directed command retry = 5
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	mon warn on pool no app = False
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	ms inject socket failures = 5000
2024-05-17T14:55:30.844 INFO:teuthology.orchestra.run.smithi171.stdout:	fsid = 81895d15-a91d-4358-a42f-b54a4e77bb53
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:[osd]
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd journal size = 100
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd scrub load threshold = 5.0
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd scrub max interval = 600
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd mclock profile = high_recovery_ops
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd recover clone overlap = true
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd recovery max chunk = 1048576
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug shutdown = true
2024-05-17T14:55:30.845 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug op order = true
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug verify stray on activate = true
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug trim objects = true
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	osd open classes on start = true
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug pg log writeout = true
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	osd deep scrub update digest min age = 30
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	osd map max advance = 10
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	journal zero on create = true
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	filestore ondisk finisher threads = 3
2024-05-17T14:55:30.846 INFO:teuthology.orchestra.run.smithi171.stdout:	filestore apply finisher threads = 3
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	bdev debug aio = true
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	osd debug misdirected ops = true
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	bdev async discard = True
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	bdev enable discard = True
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	bluestore allocator = stupid
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	bluestore block size = 96636764160
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	bluestore fsck on mount = True
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	debug bluefs = 1/20
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	debug bluestore = 1/20
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	debug ms = 1
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	debug osd = 20
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	debug rocksdb = 4/10
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd backfillfull_ratio = 0.85
2024-05-17T14:55:30.847 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd full ratio = 0.9
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd nearfull ratio = 0.8
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	osd failsafe full ratio = 0.95
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	osd objectstore = bluestore
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:[mgr]
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	debug ms = 1
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	debug mgr = 20
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	debug mon = 20
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	debug auth = 20
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:	mgr/telemetry/nag = false
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.848 INFO:teuthology.orchestra.run.smithi171.stdout:[mon]
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	debug ms = 1
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	debug mon = 20
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	debug paxos = 20
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	debug auth = 20
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	mon data avail warn = 5
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	mon mgr mkfs grace = 240
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min pgs per osd = 4
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd reporter subtree level = osd
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	mon osd prime pg temp = true
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	mon reweight min bytes per osd = 10
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	# rotate auth tickets quickly to exercise renewal paths
2024-05-17T14:55:30.849 INFO:teuthology.orchestra.run.smithi171.stdout:	auth mon ticket ttl = 660	# 11m
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	auth service ticket ttl = 240	# 4m
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	# don't complain about insecure global_id in the test suite
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_insecure_global_id_reclaim = false
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_insecure_global_id_reclaim_allowed = false
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	# 1m isn't quite enough
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_down_mkfs_grace = 2m
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	mon_warn_on_filestore_osds = false
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:[client]
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	rgw cache enabled = true
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	rgw enable ops log = true
2024-05-17T14:55:30.850 INFO:teuthology.orchestra.run.smithi171.stdout:	rgw enable usage log = true
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	log file = /var/log/ceph/$cluster-$name.$pid.log
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/$cluster-$name.$pid.asok
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	debug rbd = 20
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	debug rbd_mirror = 15
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	log to stderr = False
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	rbd default features = 125
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	rbd mirror image policy type = none
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:[mon.a]
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.0]
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.1]
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.2]
2024-05-17T14:55:30.851 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.852 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.852 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.3]
2024-05-17T14:55:30.852 INFO:teuthology.orchestra.run.smithi171.stdout:	admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok
2024-05-17T14:55:30.852 INFO:teuthology.orchestra.run.smithi171.stdout:	pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid
2024-05-17T14:55:30.853 INFO:tasks.ceph:Creating admin key on cluster2.mon.a...
2024-05-17T14:55:30.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring
2024-05-17T14:55:30.943 INFO:tasks.ceph:Copying monmap to all nodes...
2024-05-17T14:55:30.943 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:30.944 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout
2024-05-17T14:55:30.949 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:30.949 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout
2024-05-17T14:55:30.996 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi072.front.sepia.ceph.com
2024-05-17T14:55:30.996 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:30.997 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd of=/etc/ceph/cluster2.keyring
2024-05-17T14:55:30.997 DEBUG:teuthology.orchestra.run.smithi072:> sudo chmod 0644 /etc/ceph/cluster2.keyring
2024-05-17T14:55:31.021 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:31.021 DEBUG:teuthology.orchestra.run.smithi072:> dd of=/home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:31.069 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:55:31.069 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:31.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/cluster2.keyring
2024-05-17T14:55:31.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 0644 /etc/ceph/cluster2.keyring
2024-05-17T14:55:31.092 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:31.093 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:31.140 INFO:tasks.ceph:Setting up mon nodes...
2024-05-17T14:55:31.140 INFO:tasks.ceph:Setting up mgr nodes...
2024-05-17T14:55:31.141 DEBUG:teuthology.orchestra.run.smithi072:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring
2024-05-17T14:55:31.185 INFO:teuthology.orchestra.run.smithi072.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring
2024-05-17T14:55:31.188 INFO:tasks.ceph:Setting up mds nodes...
2024-05-17T14:55:31.188 INFO:tasks.ceph_client:Setting up client nodes...
2024-05-17T14:55:31.188 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring
2024-05-17T14:55:31.264 INFO:teuthology.orchestra.run.smithi072.stdout:creating /etc/ceph/cluster2.client.0.keyring
2024-05-17T14:55:31.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring
2024-05-17T14:55:31.310 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.keyring
2024-05-17T14:55:31.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring
2024-05-17T14:55:31.396 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring
2024-05-17T14:55:31.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring
2024-05-17T14:55:31.478 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring
2024-05-17T14:55:31.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring
2024-05-17T14:55:31.561 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring
2024-05-17T14:55:31.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring
2024-05-17T14:55:31.649 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring
2024-05-17T14:55:31.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring
2024-05-17T14:55:31.731 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring
2024-05-17T14:55:31.742 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring
2024-05-17T14:55:31.815 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring
2024-05-17T14:55:31.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring
2024-05-17T14:55:31.896 INFO:teuthology.orchestra.run.smithi171.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring
2024-05-17T14:55:31.906 INFO:tasks.ceph:Running mkfs on osd nodes...
2024-05-17T14:55:31.906 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi072.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi171.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}}
2024-05-17T14:55:31.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0
2024-05-17T14:55:31.958 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:55:31.959 INFO:tasks.ceph:role: cluster2.osd.0
2024-05-17T14:55:31.959 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:55:31.959 DEBUG:teuthology.orchestra.run.smithi171:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1
2024-05-17T14:55:32.019 INFO:teuthology.orchestra.run.smithi171.stdout:meta-data=/dev/vg_nvme/lv_1      isize=2048   agcount=4, agsize=5859072 blks
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sectsz=512   attr=2, projid32bit=1
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       crc=1        finobt=1, sparse=1, rmapbt=0
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       reflink=1    bigtime=0 inobtcount=0
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:data     =                       bsize=4096   blocks=23436288, imaxpct=25
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sunit=0      swidth=0 blks
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:log      =internal log           bsize=4096   blocks=11443, version=2
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sectsz=512   sunit=0 blks, lazy-count=1
2024-05-17T14:55:32.020 INFO:teuthology.orchestra.run.smithi171.stdout:realtime =none                   extsz=4096   blocks=0, rtextents=0
2024-05-17T14:55:32.484 INFO:teuthology.orchestra.run.smithi171.stdout:Discarding blocks...Done.
2024-05-17T14:55:32.486 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi171.front.sepia.ceph.com -o noatime
2024-05-17T14:55:32.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0
2024-05-17T14:55:32.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0
2024-05-17T14:55:32.654 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:55:32.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: /sbin/restorecon: command not found
2024-05-17T14:55:32.655 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1
2024-05-17T14:55:32.711 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:55:32.712 INFO:tasks.ceph:role: cluster2.osd.1
2024-05-17T14:55:32.712 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:55:32.712 DEBUG:teuthology.orchestra.run.smithi171:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2
2024-05-17T14:55:32.774 INFO:teuthology.orchestra.run.smithi171.stdout:meta-data=/dev/vg_nvme/lv_2      isize=2048   agcount=4, agsize=5859072 blks
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sectsz=512   attr=2, projid32bit=1
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       crc=1        finobt=1, sparse=1, rmapbt=0
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       reflink=1    bigtime=0 inobtcount=0
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:data     =                       bsize=4096   blocks=23436288, imaxpct=25
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sunit=0      swidth=0 blks
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:log      =internal log           bsize=4096   blocks=11443, version=2
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sectsz=512   sunit=0 blks, lazy-count=1
2024-05-17T14:55:32.775 INFO:teuthology.orchestra.run.smithi171.stdout:realtime =none                   extsz=4096   blocks=0, rtextents=0
2024-05-17T14:55:33.219 INFO:teuthology.orchestra.run.smithi171.stdout:Discarding blocks...Done.
2024-05-17T14:55:33.221 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi171.front.sepia.ceph.com -o noatime
2024-05-17T14:55:33.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1
2024-05-17T14:55:33.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1
2024-05-17T14:55:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:55:33.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: /sbin/restorecon: command not found
2024-05-17T14:55:33.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2
2024-05-17T14:55:33.397 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}
2024-05-17T14:55:33.397 INFO:tasks.ceph:role: cluster2.osd.2
2024-05-17T14:55:33.397 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:55:33.397 DEBUG:teuthology.orchestra.run.smithi171:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:meta-data=/dev/vg_nvme/lv_3      isize=2048   agcount=4, agsize=5859072 blks
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sectsz=512   attr=2, projid32bit=1
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       crc=1        finobt=1, sparse=1, rmapbt=0
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       reflink=1    bigtime=0 inobtcount=0
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:data     =                       bsize=4096   blocks=23436288, imaxpct=25
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sunit=0      swidth=0 blks
2024-05-17T14:55:33.467 INFO:teuthology.orchestra.run.smithi171.stdout:naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
2024-05-17T14:55:33.468 INFO:teuthology.orchestra.run.smithi171.stdout:log      =internal log           bsize=4096   blocks=11443, version=2
2024-05-17T14:55:33.468 INFO:teuthology.orchestra.run.smithi171.stdout:         =                       sectsz=512   sunit=0 blks, lazy-count=1
2024-05-17T14:55:33.468 INFO:teuthology.orchestra.run.smithi171.stdout:realtime =none                   extsz=4096   blocks=0, rtextents=0
2024-05-17T14:55:33.927 INFO:teuthology.orchestra.run.smithi171.stdout:Discarding blocks...Done.
2024-05-17T14:55:33.929 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi171.front.sepia.ceph.com -o noatime
2024-05-17T14:55:33.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2
2024-05-17T14:55:34.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2
2024-05-17T14:55:34.056 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:55:34.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: /sbin/restorecon: command not found
2024-05-17T14:55:34.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:34.171 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:34.169+0000 7ffb67b4e7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory
2024-05-17T14:55:34.175 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:34.169+0000 7ffb67b4e7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring
2024-05-17T14:55:34.175 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:34.169+0000 7ffb67b4e7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-0/block: (2) No such file or directory
2024-05-17T14:55:34.175 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:34.169+0000 7ffb67b4e7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-0/block: (2) No such file or directory
2024-05-17T14:55:34.176 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:34.169+0000 7ffb67b4e7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-0/block: (2) No such file or directory
2024-05-17T14:55:34.182 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:34.177+0000 7ffb67b4e7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid
2024-05-17T14:55:37.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0
2024-05-17T14:55:37.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:37.265 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:37.261+0000 7f190dd027c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory
2024-05-17T14:55:37.266 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:37.261+0000 7f190dd027c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring
2024-05-17T14:55:37.266 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:37.261+0000 7f190dd027c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-1/block: (2) No such file or directory
2024-05-17T14:55:37.266 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:37.261+0000 7f190dd027c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-1/block: (2) No such file or directory
2024-05-17T14:55:37.266 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:37.261+0000 7f190dd027c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-1/block: (2) No such file or directory
2024-05-17T14:55:37.266 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:37.261+0000 7f190dd027c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid
2024-05-17T14:55:40.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1
2024-05-17T14:55:40.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:40.395 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:40.389+0000 7f220df097c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory
2024-05-17T14:55:40.395 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:40.393+0000 7f220df097c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring
2024-05-17T14:55:40.395 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:40.393+0000 7f220df097c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-2/block: (2) No such file or directory
2024-05-17T14:55:40.396 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:40.393+0000 7f220df097c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-2/block: (2) No such file or directory
2024-05-17T14:55:40.396 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:40.393+0000 7f220df097c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/cluster2-2/block: (2) No such file or directory
2024-05-17T14:55:40.396 INFO:teuthology.orchestra.run.smithi171.stderr:2024-05-17T14:55:40.393+0000 7f220df097c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid
2024-05-17T14:55:43.371 DEBUG:teuthology.orchestra.run.smithi171:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2
2024-05-17T14:55:43.429 INFO:tasks.ceph:Reading keys from all nodes...
2024-05-17T14:55:43.429 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:43.429 DEBUG:teuthology.orchestra.run.smithi072:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout
2024-05-17T14:55:43.444 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.444 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout
2024-05-17T14:55:43.488 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.488 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout
2024-05-17T14:55:43.542 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout
2024-05-17T14:55:43.595 DEBUG:teuthology.orchestra.run.smithi072:> set -ex
2024-05-17T14:55:43.595 DEBUG:teuthology.orchestra.run.smithi072:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout
2024-05-17T14:55:43.601 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.601 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout
2024-05-17T14:55:43.644 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.644 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout
2024-05-17T14:55:43.699 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.700 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout
2024-05-17T14:55:43.750 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.751 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout
2024-05-17T14:55:43.796 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.796 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout
2024-05-17T14:55:43.844 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.845 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout
2024-05-17T14:55:43.893 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.893 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout
2024-05-17T14:55:43.940 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:43.940 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout
2024-05-17T14:55:43.988 INFO:tasks.ceph:Adding keys to all mons...
2024-05-17T14:55:43.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo tee -a /etc/ceph/cluster2.keyring
2024-05-17T14:55:44.042 INFO:teuthology.orchestra.run.smithi171.stdout:[mgr.x]
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmSlFHCxAAKf4LH8cC8Ys2WQv+5KexAA==
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:[osd.0]
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDmb0dm+qNTChAAFSTh8OTc/lkZDJSo0MCX6A==
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:[osd.1]
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDpb0dm36PpDxAAaHfj6q4rWASUrOZ5SDCTMw==
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:[osd.2]
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDsb0dmZbylFxAAyuyMdLTOAhLUwxFGNVbFJg==
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:[client.0]
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmt3YAEBAAv29jJDS8uvnVnz1v1zD91g==
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror]
2024-05-17T14:55:44.043 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmDQiUEhAATQXK4lVyKuezsbLIAVj49Q==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.0]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmdb20FxAAqZKmVTFE8PL6s3E2scCaqA==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.1]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dm/reYHBAAEHONhSsdVNxnhk8+xkPoeg==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.2]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dm/Y6OIRAARRjJyUZvEjxHpJSJufUo8w==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.3]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmua7FJhAAOFYP9uciSSCw+2r9wzv02A==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.4]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmqOKyKxAATyrS2s0P+BlcW259cSR32Q==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.5]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmjgSvMBAAAXzIo2p7z/BMklwKiKdm9w==
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.6]
2024-05-17T14:55:44.044 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmFZB9NRAAtkXL3RkINx9wDio55ei6qw==
2024-05-17T14:55:44.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *'
2024-05-17T14:55:44.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
2024-05-17T14:55:44.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
2024-05-17T14:55:44.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
2024-05-17T14:55:44.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.740 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:44.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow
2024-05-17T14:55:45.007 INFO:tasks.ceph:Running mkfs on mon nodes...
2024-05-17T14:55:45.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a
2024-05-17T14:55:45.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring
2024-05-17T14:55:45.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a
2024-05-17T14:55:45.119 DEBUG:teuthology.orchestra.run.smithi171:> rm -- /home/ubuntu/cephtest/cluster2.monmap
2024-05-17T14:55:45.167 INFO:tasks.ceph:Starting mon daemons in cluster cluster2...
2024-05-17T14:55:45.167 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon
2024-05-17T14:55:45.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a
2024-05-17T14:55:45.213 INFO:tasks.ceph.cluster2.mon.a:Started
2024-05-17T14:55:45.213 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2...
2024-05-17T14:55:45.213 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon
2024-05-17T14:55:45.213 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x
2024-05-17T14:55:45.218 INFO:tasks.ceph.cluster2.mgr.x:Started
2024-05-17T14:55:45.218 INFO:tasks.ceph:Setting crush tunables to default
2024-05-17T14:55:45.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo ceph --cluster cluster2 osd crush tunables default
2024-05-17T14:55:45.463 INFO:teuthology.orchestra.run.smithi171.stderr:adjusted tunables profile to default
2024-05-17T14:55:45.480 INFO:tasks.ceph:check_enable_crimson: False
2024-05-17T14:55:45.481 INFO:tasks.ceph:Starting osd daemons in cluster cluster2...
2024-05-17T14:55:45.481 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:45.481 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout
2024-05-17T14:55:45.492 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:45.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout
2024-05-17T14:55:45.546 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:55:45.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout
2024-05-17T14:55:45.590 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:45.590+0000 7f627b207180 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member
2024-05-17T14:55:45.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo ceph --cluster cluster2 osd new f178f5db-acb9-4423-893a-38c929460134 0
2024-05-17T14:55:45.847 INFO:teuthology.orchestra.run.smithi171.stdout:0
2024-05-17T14:55:45.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo ceph --cluster cluster2 osd new 9d580217-1539-4a3c-8725-485079ba0e8c 1
2024-05-17T14:55:45.988 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:45.990+0000 7f627b207180 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member
2024-05-17T14:55:46.066 INFO:teuthology.orchestra.run.smithi171.stdout:1
2024-05-17T14:55:46.083 DEBUG:teuthology.orchestra.run.smithi171:> sudo ceph --cluster cluster2 osd new f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0 2
2024-05-17T14:55:46.118 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.118+0000 7f627b207180 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member
2024-05-17T14:55:46.285 INFO:teuthology.orchestra.run.smithi171.stdout:2
2024-05-17T14:55:46.303 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon
2024-05-17T14:55:46.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0
2024-05-17T14:55:46.306 INFO:tasks.ceph.cluster2.osd.0:Started
2024-05-17T14:55:46.306 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon
2024-05-17T14:55:46.306 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1
2024-05-17T14:55:46.308 INFO:tasks.ceph.cluster2.osd.1:Started
2024-05-17T14:55:46.308 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon
2024-05-17T14:55:46.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2
2024-05-17T14:55:46.310 INFO:tasks.ceph.cluster2.osd.2:Started
2024-05-17T14:55:46.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json
2024-05-17T14:55:46.493 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:55:46.493 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":5,"fsid":"81895d15-a91d-4358-a42f-b54a4e77bb53","created":"2024-05-17T14:55:45.350473+0000","modified":"2024-05-17T14:55:46.286098+0000","last_up_change":"0.000000","last_in_change":"2024-05-17T14:55:46.286098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f178f5db-acb9-4423-893a-38c929460134","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"9d580217-1539-4a3c-8725-485079ba0e8c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:46.510 INFO:tasks.ceph.ceph_manager.cluster2:[]
2024-05-17T14:55:46.510 INFO:tasks.ceph:Waiting for OSDs to come up
2024-05-17T14:55:46.530 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.530+0000 7f627b207180 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member
2024-05-17T14:55:46.582 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.582+0000 7f627b207180 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member
2024-05-17T14:55:46.630 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.630+0000 7f627b207180 -1 mgr[py] Module progress has missing NOTIFY_TYPES member
2024-05-17T14:55:46.704 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.706+0000 7f627b207180 -1 mgr[py] Module crash has missing NOTIFY_TYPES member
2024-05-17T14:55:46.753 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.754+0000 7f627b207180 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member
2024-05-17T14:55:46.812 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json
2024-05-17T14:55:46.826 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.826+0000 7f627b207180 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member
2024-05-17T14:55:46.876 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:46.878+0000 7f627b207180 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member
2024-05-17T14:55:46.916 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T14:55:46.913+0000 7f28c7fd47c0 -1 Falling back to public interface
2024-05-17T14:55:46.920 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T14:55:46.917+0000 7fc8a884c7c0 -1 Falling back to public interface
2024-05-17T14:55:46.928 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T14:55:46.925+0000 7f9068bfe7c0 -1 Falling back to public interface
2024-05-17T14:55:46.996 INFO:teuthology.misc.health.smithi171.stdout:
2024-05-17T14:55:46.996 INFO:teuthology.misc.health.smithi171.stdout:{"epoch":5,"fsid":"81895d15-a91d-4358-a42f-b54a4e77bb53","created":"2024-05-17T14:55:45.350473+0000","modified":"2024-05-17T14:55:46.286098+0000","last_up_change":"0.000000","last_in_change":"2024-05-17T14:55:46.286098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f178f5db-acb9-4423-893a-38c929460134","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"9d580217-1539-4a3c-8725-485079ba0e8c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:47.013 DEBUG:teuthology.misc:0 of 3 OSDs are up
2024-05-17T14:55:47.013 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.014+0000 7f627b207180 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member
2024-05-17T14:55:47.064 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.066+0000 7f627b207180 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member
2024-05-17T14:55:47.282 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode.
2024-05-17T14:55:47.282 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve.
2024-05-17T14:55:47.282 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:  from numpy import show_config as show_numpy_config
2024-05-17T14:55:47.289 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.290+0000 7f627b207180 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member
2024-05-17T14:55:47.421 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.422+0000 7f627b207180 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member
2024-05-17T14:55:47.530 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.530+0000 7f627b207180 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member
2024-05-17T14:55:47.591 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.590+0000 7f627b207180 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member
2024-05-17T14:55:47.723 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:47.722+0000 7f627b207180 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member
2024-05-17T14:55:48.197 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:48.198+0000 7f627b207180 -1 mgr[py] Module influx has missing NOTIFY_TYPES member
2024-05-17T14:55:48.269 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:48.270+0000 7f627b207180 -1 mgr[py] Module status has missing NOTIFY_TYPES member
2024-05-17T14:55:48.412 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:48.410+0000 7f627b207180 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member
2024-05-17T14:55:48.484 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:48.482+0000 7f627b207180 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member
2024-05-17T14:55:48.764 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:48.766+0000 7f627b207180 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member
2024-05-17T14:55:48.819 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:48.818+0000 7f627b207180 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member
2024-05-17T14:55:50.054 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T14:55:50.054+0000 7f6221e48640 -1 mgr.server handle_report got status from non-daemon mon.a
2024-05-17T14:55:50.515 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T14:55:50.509+0000 7f28c7fd47c0 -1 osd.2 0 log_to_monitors true
2024-05-17T14:55:50.659 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T14:55:50.657+0000 7f9068bfe7c0 -1 osd.1 0 log_to_monitors true
2024-05-17T14:55:50.755 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T14:55:50.753+0000 7fc8a884c7c0 -1 osd.0 0 log_to_monitors true
2024-05-17T14:55:53.316 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json
2024-05-17T14:55:53.748 INFO:teuthology.misc.health.smithi171.stdout:
2024-05-17T14:55:53.748 INFO:teuthology.misc.health.smithi171.stdout:{"epoch":7,"fsid":"81895d15-a91d-4358-a42f-b54a4e77bb53","created":"2024-05-17T14:55:45.350473+0000","modified":"2024-05-17T14:55:51.899664+0000","last_up_change":"0.000000","last_in_change":"2024-05-17T14:55:46.286098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f178f5db-acb9-4423-893a-38c929460134","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"9d580217-1539-4a3c-8725-485079ba0e8c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:55:53.770 DEBUG:teuthology.misc:0 of 3 OSDs are up
2024-05-17T14:55:53.860 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T14:55:53.857+0000 7f28c3f7d640 -1 osd.2 0 waiting for initial osdmap
2024-05-17T14:55:53.892 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T14:55:53.889+0000 7f28bbdb1640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory
2024-05-17T14:55:54.139 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T14:55:53.897+0000 7f9064ba7640 -1 osd.1 0 waiting for initial osdmap
2024-05-17T14:55:54.140 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T14:55:53.897+0000 7fc8a5007640 -1 osd.0 0 waiting for initial osdmap
2024-05-17T14:55:54.140 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T14:55:53.921+0000 7fc89c629640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory
2024-05-17T14:55:54.140 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T14:55:53.921+0000 7f905c9db640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory
2024-05-17T14:56:00.073 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json
2024-05-17T14:56:00.376 INFO:teuthology.misc.health.smithi171.stdout:
2024-05-17T14:56:00.376 INFO:teuthology.misc.health.smithi171.stdout:{"epoch":12,"fsid":"81895d15-a91d-4358-a42f-b54a4e77bb53","created":"2024-05-17T14:55:45.350473+0000","modified":"2024-05-17T14:55:58.934952+0000","last_up_change":"2024-05-17T14:55:54.910897+0000","last_in_change":"2024-05-17T14:55:46.286098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-17T14:55:56.892869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f178f5db-acb9-4423-893a-38c929460134","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6809","nonce":2750238480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6811","nonce":2750238480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6815","nonce":2750238480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6813","nonce":2750238480}]},"public_addr":"172.21.15.171:6809/2750238480","cluster_addr":"172.21.15.171:6811/2750238480","heartbeat_back_addr":"172.21.15.171:6815/2750238480","heartbeat_front_addr":"172.21.15.171:6813/2750238480","state":["exists","up"]},{"osd":1,"uuid":"9d580217-1539-4a3c-8725-485079ba0e8c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6817","nonce":3577935618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6819","nonce":3577935618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6823","nonce":3577935618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6821","nonce":3577935618}]},"public_addr":"172.21.15.171:6817/3577935618","cluster_addr":"172.21.15.171:6819/3577935618","heartbeat_back_addr":"172.21.15.171:6823/3577935618","heartbeat_front_addr":"172.21.15.171:6821/3577935618","state":["exists","up"]},{"osd":2,"uuid":"f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6801","nonce":1105617501}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6803","nonce":1105617501}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6807","nonce":1105617501}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6805","nonce":1105617501}]},"public_addr":"172.21.15.171:6801/1105617501","cluster_addr":"172.21.15.171:6803/1105617501","heartbeat_back_addr":"172.21.15.171:6807/1105617501","heartbeat_front_addr":"172.21.15.171:6805/1105617501","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.716169+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.684294+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.474766+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:56:00.393 DEBUG:teuthology.misc:3 of 3 OSDs are up
2024-05-17T14:56:00.393 INFO:tasks.ceph:Creating RBD pool
2024-05-17T14:56:00.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo ceph --cluster cluster2 osd pool create rbd 8
2024-05-17T14:56:00.950 INFO:teuthology.orchestra.run.smithi171.stderr:pool 'rbd' created
2024-05-17T14:56:00.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo ceph --cluster cluster2 osd pool application enable rbd rbd --yes-i-really-mean-it
2024-05-17T14:56:01.951 INFO:teuthology.orchestra.run.smithi171.stderr:enabled application 'rbd' on pool 'rbd'
2024-05-17T14:56:01.982 INFO:tasks.ceph:Starting mds daemons in cluster cluster2...
2024-05-17T14:56:01.983 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean...
2024-05-17T14:56:01.983 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available
2024-05-17T14:56:01.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json
2024-05-17T14:56:01.983 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting
2024-05-17T14:56:02.319 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:56:02.335 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.72:6826","nonce":2408552583},{"type":"v1","addr":"172.21.15.72:6827","nonce":2408552583}]},"active_addr":"172.21.15.72:6827/2408552583","active_change":"2024-05-17T14:55:48.839269+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace.  Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB.  Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":2259185806}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":1111779132}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":3877314891}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.72:0","nonce":3731499262}]}]}
2024-05-17T14:56:02.337 INFO:tasks.ceph.ceph_manager.cluster2:mgr available!
2024-05-17T14:56:02.337 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up
2024-05-17T14:56:02.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json
2024-05-17T14:56:02.604 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:56:02.604 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":14,"fsid":"81895d15-a91d-4358-a42f-b54a4e77bb53","created":"2024-05-17T14:55:45.350473+0000","modified":"2024-05-17T14:56:01.951374+0000","last_up_change":"2024-05-17T14:55:54.910897+0000","last_in_change":"2024-05-17T14:55:46.286098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-17T14:55:56.892869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-05-17T14:56:00.684380+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f178f5db-acb9-4423-893a-38c929460134","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6809","nonce":2750238480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6811","nonce":2750238480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6815","nonce":2750238480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6813","nonce":2750238480}]},"public_addr":"172.21.15.171:6809/2750238480","cluster_addr":"172.21.15.171:6811/2750238480","heartbeat_back_addr":"172.21.15.171:6815/2750238480","heartbeat_front_addr":"172.21.15.171:6813/2750238480","state":["exists","up"]},{"osd":1,"uuid":"9d580217-1539-4a3c-8725-485079ba0e8c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6817","nonce":3577935618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6819","nonce":3577935618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6823","nonce":3577935618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6821","nonce":3577935618}]},"public_addr":"172.21.15.171:6817/3577935618","cluster_addr":"172.21.15.171:6819/3577935618","heartbeat_back_addr":"172.21.15.171:6823/3577935618","heartbeat_front_addr":"172.21.15.171:6821/3577935618","state":["exists","up"]},{"osd":2,"uuid":"f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6801","nonce":1105617501}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6803","nonce":1105617501}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6807","nonce":1105617501}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6805","nonce":1105617501}]},"public_addr":"172.21.15.171:6801/1105617501","cluster_addr":"172.21.15.171:6803/1105617501","heartbeat_back_addr":"172.21.15.171:6807/1105617501","heartbeat_front_addr":"172.21.15.171:6805/1105617501","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.716169+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.684294+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.474766+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:56:02.621 INFO:tasks.ceph.ceph_manager.cluster2:all up!
2024-05-17T14:56:02.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json
2024-05-17T14:56:02.906 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:56:02.906 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":14,"fsid":"81895d15-a91d-4358-a42f-b54a4e77bb53","created":"2024-05-17T14:55:45.350473+0000","modified":"2024-05-17T14:56:01.951374+0000","last_up_change":"2024-05-17T14:55:54.910897+0000","last_in_change":"2024-05-17T14:55:46.286098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-17T14:55:56.892869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-05-17T14:56:00.684380+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f178f5db-acb9-4423-893a-38c929460134","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6809","nonce":2750238480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6811","nonce":2750238480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6815","nonce":2750238480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2750238480},{"type":"v1","addr":"172.21.15.171:6813","nonce":2750238480}]},"public_addr":"172.21.15.171:6809/2750238480","cluster_addr":"172.21.15.171:6811/2750238480","heartbeat_back_addr":"172.21.15.171:6815/2750238480","heartbeat_front_addr":"172.21.15.171:6813/2750238480","state":["exists","up"]},{"osd":1,"uuid":"9d580217-1539-4a3c-8725-485079ba0e8c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6817","nonce":3577935618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6819","nonce":3577935618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6823","nonce":3577935618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":3577935618},{"type":"v1","addr":"172.21.15.171:6821","nonce":3577935618}]},"public_addr":"172.21.15.171:6817/3577935618","cluster_addr":"172.21.15.171:6819/3577935618","heartbeat_back_addr":"172.21.15.171:6823/3577935618","heartbeat_front_addr":"172.21.15.171:6821/3577935618","state":["exists","up"]},{"osd":2,"uuid":"f3caa4b4-d509-4559-8aec-fe5ca3b1c0f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6801","nonce":1105617501}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6803","nonce":1105617501}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6807","nonce":1105617501}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":1105617501},{"type":"v1","addr":"172.21.15.171:6805","nonce":1105617501}]},"public_addr":"172.21.15.171:6801/1105617501","cluster_addr":"172.21.15.171:6803/1105617501","heartbeat_back_addr":"172.21.15.171:6807/1105617501","heartbeat_front_addr":"172.21.15.171:6805/1105617501","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.716169+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.684294+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-17T14:55:51.474766+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
2024-05-17T14:56:02.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats
2024-05-17T14:56:02.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats
2024-05-17T14:56:02.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats
2024-05-17T14:56:03.073 INFO:teuthology.orchestra.run.smithi171.stdout:34359738374
2024-05-17T14:56:03.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0
2024-05-17T14:56:03.090 INFO:teuthology.orchestra.run.smithi171.stdout:34359738375
2024-05-17T14:56:03.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2
2024-05-17T14:56:03.091 INFO:teuthology.orchestra.run.smithi171.stdout:34359738375
2024-05-17T14:56:03.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1
2024-05-17T14:56:03.355 INFO:teuthology.orchestra.run.smithi171.stdout:34359738373
2024-05-17T14:56:03.372 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738374 got 34359738373 for osd.0
2024-05-17T14:56:03.413 INFO:teuthology.orchestra.run.smithi171.stdout:34359738374
2024-05-17T14:56:03.423 INFO:teuthology.orchestra.run.smithi171.stdout:34359738374
2024-05-17T14:56:03.435 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738375 got 34359738374 for osd.2
2024-05-17T14:56:03.443 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738375 got 34359738374 for osd.1
2024-05-17T14:56:04.373 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0
2024-05-17T14:56:04.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2
2024-05-17T14:56:04.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1
2024-05-17T14:56:04.690 INFO:teuthology.orchestra.run.smithi171.stdout:34359738373
2024-05-17T14:56:04.708 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738374 got 34359738373 for osd.0
2024-05-17T14:56:04.735 INFO:teuthology.orchestra.run.smithi171.stdout:34359738374
2024-05-17T14:56:04.752 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738375 got 34359738374 for osd.2
2024-05-17T14:56:04.763 INFO:teuthology.orchestra.run.smithi171.stdout:34359738374
2024-05-17T14:56:04.780 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738375 got 34359738374 for osd.1
2024-05-17T14:56:05.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0
2024-05-17T14:56:05.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2
2024-05-17T14:56:05.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1
2024-05-17T14:56:06.010 INFO:teuthology.orchestra.run.smithi171.stdout:34359738374
2024-05-17T14:56:06.033 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738374 got 34359738374 for osd.0
2024-05-17T14:56:06.033 DEBUG:teuthology.parallel:result is None
2024-05-17T14:56:06.064 INFO:teuthology.orchestra.run.smithi171.stdout:34359738375
2024-05-17T14:56:06.081 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738375 got 34359738375 for osd.2
2024-05-17T14:56:06.081 DEBUG:teuthology.parallel:result is None
2024-05-17T14:56:06.086 INFO:teuthology.orchestra.run.smithi171.stdout:34359738375
2024-05-17T14:56:06.102 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738375 got 34359738375 for osd.1
2024-05-17T14:56:06.102 DEBUG:teuthology.parallel:result is None
2024-05-17T14:56:06.102 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean
2024-05-17T14:56:06.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json
2024-05-17T14:56:06.421 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:56:06.422 INFO:teuthology.orchestra.run.smithi171.stderr:dumped all
2024-05-17T14:56:06.438 INFO:teuthology.orchestra.run.smithi171.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-05-17T14:56:04.852548+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":7,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82580,"kb_used_data":1520,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032940,"statfs":{"total":289910292480,"available":289825730560,"internally_reserved":0,"allocated":1556480,"data_stored":1282004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.892170"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.968109+0000","last_change":"2024-05-17T14:56:01.965686+0000","last_active":"2024-05-17T14:56:01.968109+0000","last_peered":"2024-05-17T14:56:01.968109+0000","last_clean":"2024-05-17T14:56:01.968109+0000","last_became_active":"2024-05-17T14:56:01.965274+0000","last_became_peered":"2024-05-17T14:56:01.965274+0000","last_unstale":"2024-05-17T14:56:01.968109+0000","last_undegraded":"2024-05-17T14:56:01.968109+0000","last_fullsized":"2024-05-17T14:56:01.968109+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:29:32.920251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969456+0000","last_change":"2024-05-17T14:56:01.968452+0000","last_active":"2024-05-17T14:56:01.969456+0000","last_peered":"2024-05-17T14:56:01.969456+0000","last_clean":"2024-05-17T14:56:01.969456+0000","last_became_active":"2024-05-17T14:56:01.968166+0000","last_became_peered":"2024-05-17T14:56:01.968166+0000","last_unstale":"2024-05-17T14:56:01.969456+0000","last_undegraded":"2024-05-17T14:56:01.969456+0000","last_fullsized":"2024-05-17T14:56:01.969456+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:05:47.255402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969322+0000","last_change":"2024-05-17T14:56:01.967348+0000","last_active":"2024-05-17T14:56:01.969322+0000","last_peered":"2024-05-17T14:56:01.969322+0000","last_clean":"2024-05-17T14:56:01.969322+0000","last_became_active":"2024-05-17T14:56:01.966841+0000","last_became_peered":"2024-05-17T14:56:01.966841+0000","last_unstale":"2024-05-17T14:56:01.969322+0000","last_undegraded":"2024-05-17T14:56:01.969322+0000","last_fullsized":"2024-05-17T14:56:01.969322+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T18:07:10.476590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969408+0000","last_change":"2024-05-17T14:56:01.967712+0000","last_active":"2024-05-17T14:56:01.969408+0000","last_peered":"2024-05-17T14:56:01.969408+0000","last_clean":"2024-05-17T14:56:01.969408+0000","last_became_active":"2024-05-17T14:56:01.966962+0000","last_became_peered":"2024-05-17T14:56:01.966962+0000","last_unstale":"2024-05-17T14:56:01.969408+0000","last_undegraded":"2024-05-17T14:56:01.969408+0000","last_fullsized":"2024-05-17T14:56:01.969408+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T18:08:57.411958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.967321+0000","last_change":"2024-05-17T14:56:01.963908+0000","last_active":"2024-05-17T14:56:01.967321+0000","last_peered":"2024-05-17T14:56:01.967321+0000","last_clean":"2024-05-17T14:56:01.967321+0000","last_became_active":"2024-05-17T14:56:01.963229+0000","last_became_peered":"2024-05-17T14:56:01.963229+0000","last_unstale":"2024-05-17T14:56:01.967321+0000","last_undegraded":"2024-05-17T14:56:01.967321+0000","last_fullsized":"2024-05-17T14:56:01.967321+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T23:11:42.488091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-05-17T14:56:02.963732+0000","last_change":"2024-05-17T14:56:01.965269+0000","last_active":"2024-05-17T14:56:02.963732+0000","last_peered":"2024-05-17T14:56:02.963732+0000","last_clean":"2024-05-17T14:56:02.963732+0000","last_became_active":"2024-05-17T14:56:01.964965+0000","last_became_peered":"2024-05-17T14:56:01.964965+0000","last_unstale":"2024-05-17T14:56:02.963732+0000","last_undegraded":"2024-05-17T14:56:02.963732+0000","last_fullsized":"2024-05-17T14:56:02.963732+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T23:53:49.125665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-05-17T14:56:02.963731+0000","last_change":"2024-05-17T14:56:01.963935+0000","last_active":"2024-05-17T14:56:02.963731+0000","last_peered":"2024-05-17T14:56:02.963731+0000","last_clean":"2024-05-17T14:56:02.963731+0000","last_became_active":"2024-05-17T14:56:01.963399+0000","last_became_peered":"2024-05-17T14:56:01.963399+0000","last_unstale":"2024-05-17T14:56:02.963731+0000","last_undegraded":"2024-05-17T14:56:02.963731+0000","last_fullsized":"2024-05-17T14:56:02.963731+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:51:52.670317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969399+0000","last_change":"2024-05-17T14:56:01.967840+0000","last_active":"2024-05-17T14:56:01.969399+0000","last_peered":"2024-05-17T14:56:01.969399+0000","last_clean":"2024-05-17T14:56:01.969399+0000","last_became_active":"2024-05-17T14:56:01.966546+0000","last_became_peered":"2024-05-17T14:56:01.966546+0000","last_unstale":"2024-05-17T14:56:01.969399+0000","last_undegraded":"2024-05-17T14:56:01.969399+0000","last_fullsized":"2024-05-17T14:56:01.969399+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:44:26.712124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":82,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.961853+0000","last_change":"2024-05-17T14:55:58.271760+0000","last_active":"2024-05-17T14:56:01.961853+0000","last_peered":"2024-05-17T14:56:01.961853+0000","last_clean":"2024-05-17T14:56:01.961853+0000","last_became_active":"2024-05-17T14:55:58.269413+0000","last_became_peered":"2024-05-17T14:55:58.269413+0000","last_unstale":"2024-05-17T14:56:01.961853+0000","last_undegraded":"2024-05-17T14:56:01.961853+0000","last_fullsized":"2024-05-17T14:56:01.961853+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:56.924195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:56.924195+0000","last_clean_scrub_stamp":"2024-05-17T14:55:56.924195+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T15:28:57.285251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738375,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27140,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344700,"statfs":{"total":96636764160,"available":96608972800,"internally_reserved":0,"allocated":122880,"data_stored":33756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738375,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27720,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344120,"statfs":{"total":96636764160,"available":96608378880,"internally_reserved":0,"allocated":716800,"data_stored":624124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738374,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27720,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344120,"statfs":{"total":96636764160,"available":96608378880,"internally_reserved":0,"allocated":716800,"data_stored":624124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
2024-05-17T14:56:06.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json
2024-05-17T14:56:06.724 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:56:06.724 INFO:teuthology.orchestra.run.smithi171.stderr:dumped all
2024-05-17T14:56:06.742 INFO:teuthology.orchestra.run.smithi171.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-05-17T14:56:04.852548+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":7,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82580,"kb_used_data":1520,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032940,"statfs":{"total":289910292480,"available":289825730560,"internally_reserved":0,"allocated":1556480,"data_stored":1282004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.892170"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.968109+0000","last_change":"2024-05-17T14:56:01.965686+0000","last_active":"2024-05-17T14:56:01.968109+0000","last_peered":"2024-05-17T14:56:01.968109+0000","last_clean":"2024-05-17T14:56:01.968109+0000","last_became_active":"2024-05-17T14:56:01.965274+0000","last_became_peered":"2024-05-17T14:56:01.965274+0000","last_unstale":"2024-05-17T14:56:01.968109+0000","last_undegraded":"2024-05-17T14:56:01.968109+0000","last_fullsized":"2024-05-17T14:56:01.968109+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:29:32.920251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969456+0000","last_change":"2024-05-17T14:56:01.968452+0000","last_active":"2024-05-17T14:56:01.969456+0000","last_peered":"2024-05-17T14:56:01.969456+0000","last_clean":"2024-05-17T14:56:01.969456+0000","last_became_active":"2024-05-17T14:56:01.968166+0000","last_became_peered":"2024-05-17T14:56:01.968166+0000","last_unstale":"2024-05-17T14:56:01.969456+0000","last_undegraded":"2024-05-17T14:56:01.969456+0000","last_fullsized":"2024-05-17T14:56:01.969456+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:05:47.255402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969322+0000","last_change":"2024-05-17T14:56:01.967348+0000","last_active":"2024-05-17T14:56:01.969322+0000","last_peered":"2024-05-17T14:56:01.969322+0000","last_clean":"2024-05-17T14:56:01.969322+0000","last_became_active":"2024-05-17T14:56:01.966841+0000","last_became_peered":"2024-05-17T14:56:01.966841+0000","last_unstale":"2024-05-17T14:56:01.969322+0000","last_undegraded":"2024-05-17T14:56:01.969322+0000","last_fullsized":"2024-05-17T14:56:01.969322+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T18:07:10.476590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969408+0000","last_change":"2024-05-17T14:56:01.967712+0000","last_active":"2024-05-17T14:56:01.969408+0000","last_peered":"2024-05-17T14:56:01.969408+0000","last_clean":"2024-05-17T14:56:01.969408+0000","last_became_active":"2024-05-17T14:56:01.966962+0000","last_became_peered":"2024-05-17T14:56:01.966962+0000","last_unstale":"2024-05-17T14:56:01.969408+0000","last_undegraded":"2024-05-17T14:56:01.969408+0000","last_fullsized":"2024-05-17T14:56:01.969408+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T18:08:57.411958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.967321+0000","last_change":"2024-05-17T14:56:01.963908+0000","last_active":"2024-05-17T14:56:01.967321+0000","last_peered":"2024-05-17T14:56:01.967321+0000","last_clean":"2024-05-17T14:56:01.967321+0000","last_became_active":"2024-05-17T14:56:01.963229+0000","last_became_peered":"2024-05-17T14:56:01.963229+0000","last_unstale":"2024-05-17T14:56:01.967321+0000","last_undegraded":"2024-05-17T14:56:01.967321+0000","last_fullsized":"2024-05-17T14:56:01.967321+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T23:11:42.488091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-05-17T14:56:02.963732+0000","last_change":"2024-05-17T14:56:01.965269+0000","last_active":"2024-05-17T14:56:02.963732+0000","last_peered":"2024-05-17T14:56:02.963732+0000","last_clean":"2024-05-17T14:56:02.963732+0000","last_became_active":"2024-05-17T14:56:01.964965+0000","last_became_peered":"2024-05-17T14:56:01.964965+0000","last_unstale":"2024-05-17T14:56:02.963732+0000","last_undegraded":"2024-05-17T14:56:02.963732+0000","last_fullsized":"2024-05-17T14:56:02.963732+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T23:53:49.125665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-05-17T14:56:02.963731+0000","last_change":"2024-05-17T14:56:01.963935+0000","last_active":"2024-05-17T14:56:02.963731+0000","last_peered":"2024-05-17T14:56:02.963731+0000","last_clean":"2024-05-17T14:56:02.963731+0000","last_became_active":"2024-05-17T14:56:01.963399+0000","last_became_peered":"2024-05-17T14:56:01.963399+0000","last_unstale":"2024-05-17T14:56:02.963731+0000","last_undegraded":"2024-05-17T14:56:02.963731+0000","last_fullsized":"2024-05-17T14:56:02.963731+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:51:52.670317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.969399+0000","last_change":"2024-05-17T14:56:01.967840+0000","last_active":"2024-05-17T14:56:01.969399+0000","last_peered":"2024-05-17T14:56:01.969399+0000","last_clean":"2024-05-17T14:56:01.969399+0000","last_became_active":"2024-05-17T14:56:01.966546+0000","last_became_peered":"2024-05-17T14:56:01.966546+0000","last_unstale":"2024-05-17T14:56:01.969399+0000","last_undegraded":"2024-05-17T14:56:01.969399+0000","last_fullsized":"2024-05-17T14:56:01.969399+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:56:00.945494+0000","last_clean_scrub_stamp":"2024-05-17T14:56:00.945494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-19T02:44:26.712124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":82,"reported_epoch":14,"state":"active+clean","last_fresh":"2024-05-17T14:56:01.961853+0000","last_change":"2024-05-17T14:55:58.271760+0000","last_active":"2024-05-17T14:56:01.961853+0000","last_peered":"2024-05-17T14:56:01.961853+0000","last_clean":"2024-05-17T14:56:01.961853+0000","last_became_active":"2024-05-17T14:55:58.269413+0000","last_became_peered":"2024-05-17T14:55:58.269413+0000","last_unstale":"2024-05-17T14:56:01.961853+0000","last_undegraded":"2024-05-17T14:56:01.961853+0000","last_fullsized":"2024-05-17T14:56:01.961853+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-17T14:55:56.924195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-17T14:55:56.924195+0000","last_clean_scrub_stamp":"2024-05-17T14:55:56.924195+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-18T15:28:57.285251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738375,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27140,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344700,"statfs":{"total":96636764160,"available":96608972800,"internally_reserved":0,"allocated":122880,"data_stored":33756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738375,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27720,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344120,"statfs":{"total":96636764160,"available":96608378880,"internally_reserved":0,"allocated":716800,"data_stored":624124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738374,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27720,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344120,"statfs":{"total":96636764160,"available":96608378880,"internally_reserved":0,"allocated":716800,"data_stored":624124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
2024-05-17T14:56:06.743 INFO:tasks.ceph.ceph_manager.cluster2:clean!
2024-05-17T14:56:06.743 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy...
2024-05-17T14:56:06.743 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy
2024-05-17T14:56:06.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json
2024-05-17T14:56:07.052 INFO:teuthology.orchestra.run.smithi171.stdout:
2024-05-17T14:56:07.053 INFO:teuthology.orchestra.run.smithi171.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]}
2024-05-17T14:56:07.071 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done
2024-05-17T14:56:07.071 INFO:teuthology.run_tasks:Running task exec...
2024-05-17T14:56:07.081 INFO:teuthology.task.exec:Executing custom commands...
2024-05-17T14:56:07.081 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi171.front.sepia.ceph.com
2024-05-17T14:56:07.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:07.441 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror]
2024-05-17T14:56:07.441 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQC9b0dms0kFLBAAoLxftY/5mb3ZXKGRzf9FqA==
2024-05-17T14:56:07.441 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror-peer"
2024-05-17T14:56:07.441 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:07.441 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror
2024-05-17T14:56:07.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:07.818 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.0]
2024-05-17T14:56:07.818 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQC9b0dmfpi+MhAAJ9RfgV78UVEO18VyDKH0aQ==
2024-05-17T14:56:07.818 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:07.818 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:07.818 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.0
2024-05-17T14:56:07.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:08.190 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.1]
2024-05-17T14:56:08.190 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQC9b0dmJh7TNxAAyBv11LYHGiv0t1WZkf/fGw==
2024-05-17T14:56:08.190 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:08.190 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:08.190 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.1
2024-05-17T14:56:08.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:08.578 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.2]
2024-05-17T14:56:08.578 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQC+b0dmSoMgARAAv8eHMZjUUiY3xOf81CQwlg==
2024-05-17T14:56:08.578 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:08.578 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:08.579 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.2
2024-05-17T14:56:08.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:08.970 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.3]
2024-05-17T14:56:08.971 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQC+b0dmtqtFBhAAb7TIIXQIE4gxIVPMMvFMJg==
2024-05-17T14:56:08.971 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:08.971 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:08.971 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.3
2024-05-17T14:56:08.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:09.338 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror]
2024-05-17T14:56:09.338 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmDQiUEhAATQXK4lVyKuezsbLIAVj49Q==
2024-05-17T14:56:09.338 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror-peer"
2024-05-17T14:56:09.338 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:09.338 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror
2024-05-17T14:56:09.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:09.707 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.0]
2024-05-17T14:56:09.707 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmdb20FxAAqZKmVTFE8PL6s3E2scCaqA==
2024-05-17T14:56:09.707 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:09.707 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:09.707 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.0
2024-05-17T14:56:09.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:10.078 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.1]
2024-05-17T14:56:10.078 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dm/reYHBAAEHONhSsdVNxnhk8+xkPoeg==
2024-05-17T14:56:10.078 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:10.078 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:10.078 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.1
2024-05-17T14:56:10.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:10.446 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.2]
2024-05-17T14:56:10.446 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dm/Y6OIRAARRjJyUZvEjxHpJSJufUo8w==
2024-05-17T14:56:10.446 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:10.446 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:10.447 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.2
2024-05-17T14:56:10.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"''
2024-05-17T14:56:10.814 INFO:teuthology.orchestra.run.smithi171.stdout:[client.mirror.3]
2024-05-17T14:56:10.815 INFO:teuthology.orchestra.run.smithi171.stdout:	key = AQDjb0dmua7FJhAAOFYP9uciSSCw+2r9wzv02A==
2024-05-17T14:56:10.815 INFO:teuthology.orchestra.run.smithi171.stdout:	caps mon = "profile rbd-mirror"
2024-05-17T14:56:10.815 INFO:teuthology.orchestra.run.smithi171.stdout:	caps osd = "profile rbd"
2024-05-17T14:56:10.815 INFO:teuthology.orchestra.run.smithi171.stderr:updated caps for client.mirror.3
2024-05-17T14:56:10.832 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon
2024-05-17T14:56:10.846 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0
2024-05-17T14:56:10.849 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started
2024-05-17T14:56:10.849 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.858 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon
2024-05-17T14:56:10.858 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1
2024-05-17T14:56:10.860 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started
2024-05-17T14:56:10.860 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.871 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon
2024-05-17T14:56:10.871 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2
2024-05-17T14:56:10.873 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started
2024-05-17T14:56:10.873 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.884 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon
2024-05-17T14:56:10.885 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3
2024-05-17T14:56:10.886 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started
2024-05-17T14:56:10.887 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.901 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon
2024-05-17T14:56:10.901 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0
2024-05-17T14:56:10.903 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started
2024-05-17T14:56:10.903 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.913 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon
2024-05-17T14:56:10.914 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1
2024-05-17T14:56:10.915 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started
2024-05-17T14:56:10.916 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.926 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T14:56:10.927 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T14:56:10.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T14:56:10.929 INFO:teuthology.run_tasks:Running task rbd-mirror...
2024-05-17T14:56:10.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon
2024-05-17T14:56:10.939 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3
2024-05-17T14:56:10.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started
2024-05-17T14:56:10.941 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash...
2024-05-17T14:56:10.953 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1715957770
2024-05-17T14:56:10.954 DEBUG:tasks.rbd_mirror_thrash:Yielding
2024-05-17T14:56:10.954 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash...
2024-05-17T14:56:10.964 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1715957770
2024-05-17T14:56:10.964 DEBUG:tasks.rbd_mirror_thrash:Yielding
2024-05-17T14:56:10.964 INFO:teuthology.run_tasks:Running task workunit...
2024-05-17T14:56:10.979 INFO:tasks.workunit:Pulling workunits from ref ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:56:10.979 INFO:tasks.workunit:Making a separate scratch dir for every client...
2024-05-17T14:56:10.979 DEBUG:teuthology.orchestra.run.smithi171:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror
2024-05-17T14:56:10.980 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1
2024-05-17T14:56:10.980 INFO:tasks.rbd_mirror_thrash:waiting for 68 secs before thrashing
2024-05-17T14:56:10.980 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2
2024-05-17T14:56:10.980 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing
2024-05-17T14:56:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-05-17T14:56:10.985 INFO:teuthology.orchestra.run.smithi171.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory
2024-05-17T14:56:10.985 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror
2024-05-17T14:56:11.031 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror
2024-05-17T14:56:11.031 DEBUG:teuthology.orchestra.run.smithi171:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror
2024-05-17T14:56:11.078 INFO:tasks.workunit:timeout=6h
2024-05-17T14:56:11.079 INFO:tasks.workunit:cleanup=True
2024-05-17T14:56:11.079 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout ee59d86aad04089481e55c1a15aa5871339935b4
2024-05-17T14:56:11.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'...
2024-05-17T14:56:55.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Note: switching to 'ee59d86aad04089481e55c1a15aa5871339935b4'.
2024-05-17T14:56:55.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:You are in 'detached HEAD' state. You can look around, make experimental
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:changes and commit them, and you can discard any commits you make in this
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:state without impacting any branches by switching back to a branch.
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:If you want to create a new branch to retain commits you create, you may
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:do so (now or later) by using -c with the switch command. Example:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  git switch -c <new-branch-name>
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Or undo this operation with:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  git switch -
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Turn off this advice by setting config variable advice.detachedHead to false
2024-05-17T14:56:55.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:56:55.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:HEAD is now at ee59d86aad0 Merge branch 'wip-64800' of https://github.com/ajarr/ceph into HEAD
2024-05-17T14:56:55.333 DEBUG:teuthology.orchestra.run.smithi171:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror
2024-05-17T14:56:55.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done
2024-05-17T14:56:55.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io'
2024-05-17T14:56:55.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test
2024-05-17T14:56:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io
2024-05-17T14:56:56.215 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read
2024-05-17T14:56:56.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io'
2024-05-17T14:56:56.265 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs'
2024-05-17T14:56:56.266 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc
2024-05-17T14:56:56.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs'
2024-05-17T14:56:56.313 DEBUG:teuthology.orchestra.run.smithi171:> set -ex
2024-05-17T14:56:56.314 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout
2024-05-17T14:56:56.361 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror...
2024-05-17T14:56:56.362 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh...
2024-05-17T14:56:56.363 DEBUG:teuthology.orchestra.run.smithi171:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=ee59d86aad04089481e55c1a15aa5871339935b4 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering,exclusive-lock RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh
2024-05-17T14:56:56.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ IMAGE_COUNT=50
2024-05-17T14:56:56.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ export LOCKDEP=0
2024-05-17T14:56:56.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ LOCKDEP=0
2024-05-17T14:56:56.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh
2024-05-17T14:56:56.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh
2024-05-17T14:56:56.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ type xmlstarlet
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ XMLSTARLET=xmlstarlet
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ RBD_MIRROR_INSTANCES=4
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CLUSTER1=cluster1
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CLUSTER2=cluster2
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ PEER_CLUSTER_SUFFIX=
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ POOL=mirror
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ PARENT_POOL=mirror_parent
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ NS1=ns1
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ NS2=ns2
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ TEMPDIR=
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ID=mirror
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ MIRROR_USER_ID_PREFIX=mirror.
2024-05-17T14:56:56.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ RBD_MIRROR_MODE=snapshot
2024-05-17T14:56:56.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ MIRROR_POOL_MODE=image
2024-05-17T14:56:56.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' snapshot = snapshot ']'
2024-05-17T14:56:56.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ MIRROR_POOL_MODE=image
2024-05-17T14:56:56.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ export 'CEPH_ARGS=--id mirror'
2024-05-17T14:56:56.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS='--id mirror'
2024-05-17T14:56:56.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ LAST_MIRROR_INSTANCE=3
2024-05-17T14:56:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh
2024-05-17T14:56:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src
2024-05-17T14:56:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src
2024-05-17T14:56:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_BIN=.
2024-05-17T14:56:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_SRC=.
2024-05-17T14:56:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' -e CMakeCache.txt ']'
2024-05-17T14:56:56.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 -gt 0 ']'
2024-05-17T14:56:56.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ setup
2024-05-17T14:56:56.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local c
2024-05-17T14:56:56.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ trap 'cleanup $?' INT TERM EXIT
2024-05-17T14:56:56.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ setup_tempdir
2024-05-17T14:56:56.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:56:56.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mktemp -d
2024-05-17T14:56:56.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ TEMPDIR=/tmp/tmp.mfaeqiv6jj
2024-05-17T14:56:56.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -z 1 ']'
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ setup_pools cluster1 cluster2
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local remote_cluster=cluster2
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mon_map_file
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mon_addr
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local admin_key_file
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local uuid
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:56:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64
2024-05-17T14:56:57.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:pool 'mirror' already exists
2024-05-17T14:56:57.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:56:57.760 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64
2024-05-17T14:56:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:pool 'mirror_parent' already exists
2024-05-17T14:56:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:56:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 pool init mirror
2024-05-17T14:57:01.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:01.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 pool init mirror_parent
2024-05-17T14:57:04.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:04.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:04.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image
2024-05-17T14:57:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image
2024-05-17T14:57:04.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1
2024-05-17T14:57:04.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2
2024-05-17T14:57:04.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image
2024-05-17T14:57:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image
2024-05-17T14:57:05.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -z ']'
2024-05-17T14:57:05.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -z ']'
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_add cluster1 mirror cluster2
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local client_cluster=cluster2
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local remote_cluster=cluster2
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local uuid_var_name
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local error_code
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local peer_uuid
2024-05-17T14:57:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 16 32
2024-05-17T14:57:05.060 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:57:05.060 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2
2024-05-17T14:57:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_uuid=b38475ef-ee9c-43e8-a5fc-5f468e3a0055
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=0
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 0 -eq 17 ']'
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 0 -eq 0
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_add cluster1 mirror_parent cluster2
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror_parent
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local client_cluster=cluster2
2024-05-17T14:57:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local remote_cluster=cluster2
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local uuid_var_name
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local error_code
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local peer_uuid
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 16 32
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:57:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_uuid=3d680bc2-aeba-43c8-8bee-c93718c988fb
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=0
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 0 -eq 17 ']'
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 0 -eq 0
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ setup_pools cluster2 cluster1
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local remote_cluster=cluster1
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mon_map_file
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mon_addr
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local admin_key_file
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local uuid
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64
2024-05-17T14:57:06.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:pool 'mirror' already exists
2024-05-17T14:57:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64
2024-05-17T14:57:07.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:pool 'mirror_parent' already exists
2024-05-17T14:57:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 pool init mirror
2024-05-17T14:57:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 pool init mirror_parent
2024-05-17T14:57:11.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi171.stderr:2024-05-17T14:57:11.058+0000 7f014dbd4880 -1 rbd::mirror::PoolReplayer: 0x55ff0248cc00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.081 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi171.stderr:2024-05-17T14:57:11.078+0000 7f5dfdb50880 -1 rbd::mirror::PoolReplayer: 0x55a980600c00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi171.stderr:2024-05-17T14:57:11.082+0000 7f470d8d1880 -1 rbd::mirror::PoolReplayer: 0x55692fa3cc00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T14:57:11.082+0000 7f0fdbeff880 -1 rbd::mirror::PoolReplayer: 0x564bd16acc00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi171.stderr:2024-05-17T14:57:11.114+0000 7f014dbd4880 -1 rbd::mirror::PoolReplayer: 0x55ff030d8f00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi171.stderr:2024-05-17T14:57:11.134+0000 7f5dfdb50880 -1 rbd::mirror::PoolReplayer: 0x55a9812c8f00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.145 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T14:57:11.142+0000 7f0fdbeff880 -1 rbd::mirror::PoolReplayer: 0x564bd2320f00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.349 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi171.stderr:2024-05-17T14:57:11.346+0000 7f470d8d1880 -1 rbd::mirror::PoolReplayer: 0x5569306b2f00 init: failed to initialize remote pool poller: (2) No such file or directory
2024-05-17T14:57:11.982 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.891232368059661) > weight (0.25)
2024-05-17T14:57:11.982 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6169035862946484) > weight (0.25)
2024-05-17T14:57:11.982 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T14:57:11.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T14:57:11.983 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons
2024-05-17T14:57:11.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T14:57:11.978+0000 7f017afd8640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 29323) UID: 1000
2024-05-17T14:57:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:13.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ CEPH_ARGS=
2024-05-17T14:57:13.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image
2024-05-17T14:57:13.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image
2024-05-17T14:57:13.230 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1
2024-05-17T14:57:13.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2
2024-05-17T14:57:13.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image
2024-05-17T14:57:13.347 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -z ']'
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -z ']'
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_add cluster2 mirror cluster1
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local client_cluster=cluster1
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local remote_cluster=cluster1
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local uuid_var_name
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local error_code
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local peer_uuid
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 16 32
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:57:13.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1
2024-05-17T14:57:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_uuid=aebab606-ddc6-4768-a9b8-39bb3afb4ed7
2024-05-17T14:57:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=0
2024-05-17T14:57:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:57:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 0 -eq 17 ']'
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 0 -eq 0
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_add cluster2 mirror_parent cluster1
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror_parent
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local client_cluster=cluster1
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local remote_cluster=cluster1
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local uuid_var_name
2024-05-17T14:57:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local error_code
2024-05-17T14:57:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local peer_uuid
2024-05-17T14:57:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 16 32
2024-05-17T14:57:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:57:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1
2024-05-17T14:57:13.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ peer_uuid=0d1c352b-4389-49ac-b66c-eea3753823c5
2024-05-17T14:57:13.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=0
2024-05-17T14:57:13.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:57:13.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 0 -eq 17 ']'
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 0 -eq 0
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirrors cluster1
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 0 3
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster1:0
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1:0
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster1:0
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster1
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=0
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster1
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=0
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster1:1
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1:1
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster1:1 cluster instance
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster1:1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster1:2
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1:2
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster1:2 cluster instance
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster1:2
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster1
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=2
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster1
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=2
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster1:3
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1:3
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster1:3 cluster instance
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster1:3
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster1
2024-05-17T14:57:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=3
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster1
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=3
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirrors cluster2
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 0 3
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster2:0
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2:0
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster2:0 cluster instance
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster2:0
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster2
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=0
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 0 = cluster2:0 ']'
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster2
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster2
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=0
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster2:1
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2:1
2024-05-17T14:57:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster2:1 cluster instance
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster2:1
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster2
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=1
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 1 = cluster2:1 ']'
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster2
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster2
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=1
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster2:2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2:2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster2:2 cluster instance
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster2:2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 2 = cluster2:2 ']'
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=2
2024-05-17T14:57:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ start_mirror cluster2:3
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2:3
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set_cluster_instance cluster2:3 cluster instance
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local val=cluster2:3
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster_var_name=cluster
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local instance_var_name=instance
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cluster=cluster2
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ instance=3
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 3 = cluster2:3 ']'
2024-05-17T14:57:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval cluster=cluster2
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster2
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ eval instance=3
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n 1
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ testlog 'TEST: add image and test replay after client crashes'
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ tee -a /tmp/tmp.mfaeqiv6jj/rbd-mirror.test.log
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ date '+%F %T'
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo 2024-05-17 14:57:13 TEST: add image and test replay after client crashes
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-17 14:57:13 TEST: add image and test replay after client crashes
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=test
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M
2024-05-17T14:57:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror test 512M
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T14:57:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 512M ']'
2024-05-17T14:57:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=512M
2024-05-17T14:57:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T14:57:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/test
2024-05-17T14:57:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T14:57:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror test snapshot
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T14:57:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot
2024-05-17T14:57:14.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T14:57:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/test
2024-05-17T14:57:14.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T14:57:14.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T14:57:14.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10c02955642e
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10c02955642e
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:13 2024
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 14:57:13 2024
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:13 2024
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T14:57:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:57:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T14:57:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:14.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T14:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T14:57:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T14:57:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T14:57:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:57:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:57:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:57:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:57:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:57:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:57:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:57:15.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:15.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:15.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:15.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:57:15.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:15.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:15.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:15.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T14:57:15.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:15.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:15.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T14:57:15.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T14:57:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T14:57:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T14:57:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T14:57:15.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:15.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:15.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T14:57:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T14:57:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:15.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T14:57:15.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T14:57:15.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T14:57:15.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:15.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:15.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:15.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T14:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T14:57:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T14:57:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:15.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:15.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T14:57:15.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T14:57:15.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T14:57:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T14:57:15.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:15.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:15.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:15.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:15.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 1
2024-05-17T14:57:16.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:57:16.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:57:16.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:57:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:57:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:57:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:57:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:16.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:57:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T14:57:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T14:57:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T14:57:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T14:57:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T14:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T14:57:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T14:57:17.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:17.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:17.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:17.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T14:57:17.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T14:57:17.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:17.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:17.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T14:57:17.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T14:57:17.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T14:57:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:17.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:17.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T14:57:17.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:17.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:17.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:17.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:17.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T14:57:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T14:57:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T14:57:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T14:57:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:17.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:17.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:17.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T14:57:17.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:17.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:17.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T14:57:17.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T14:57:17.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:17.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:17.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:17.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T14:57:17.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:17.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:17.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:17.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:17.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 2
2024-05-17T14:57:17.984 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T14:57:17.984 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T14:57:17.985 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T14:57:17.985 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T14:57:17.985 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T14:57:17.988 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T14:57:17.988 INFO:tasks.rbd_mirror_thrash:waiting for 77 secs before thrashing
2024-05-17T14:57:18.981 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0
2024-05-17T14:57:18.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15
2024-05-17T14:57:18.982 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons
2024-05-17T14:57:18.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T14:57:18.978+0000 7f0fd8eba640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 29307) UID: 1000
2024-05-17T14:57:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:57:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:57:19.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:19.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:57:19.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:57:19.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:19.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:19.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:57:19.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:57:19.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:19.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:19.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:19.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:57:19.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:19.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:57:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:19.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:57:19.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:57:19.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:57:19.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:57:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:19.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T14:57:19.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T14:57:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T14:57:19.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:19.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:19.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T14:57:19.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T14:57:19.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T14:57:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T14:57:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T14:57:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:19.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:19.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:19.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T14:57:19.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T14:57:19.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T14:57:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T14:57:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T14:57:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T14:57:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T14:57:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T14:57:19.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:19.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:19.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T14:57:19.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T14:57:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T14:57:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T14:57:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T14:57:19.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:19.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:19.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T14:57:19.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T14:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T14:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T14:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T14:57:19.898 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:19.898 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 4
2024-05-17T14:57:20.982 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0
2024-05-17T14:57:20.982 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T14:57:20.983 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0
2024-05-17T14:57:20.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon
2024-05-17T14:57:20.983 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0
2024-05-17T14:57:20.986 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started
2024-05-17T14:57:20.986 INFO:tasks.rbd_mirror_thrash:waiting for 87 secs before thrashing
2024-05-17T14:57:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:57:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:57:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:57:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:57:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:57:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:57:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:57:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:57:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:57:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:57:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:57:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:57:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:57:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:57:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:57:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:57:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T14:57:24.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T14:57:24.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T14:57:24.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T14:57:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T14:57:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T14:57:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T14:57:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=
2024-05-17T14:57:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 10
2024-05-17T14:57:24.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T14:57:24.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T14:57:24.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:24.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:24.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T14:57:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T14:57:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:57:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T14:57:24.213 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 37645312 bytes
2024-05-17T14:57:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 51128320 bytes
2024-05-17T14:57:24.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 103943168 bytes
2024-05-17T14:57:24.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 120688640 bytes
2024-05-17T14:57:24.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 121289728 bytes
2024-05-17T14:57:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 209129472 bytes
2024-05-17T14:57:25.026 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 240700416 bytes
2024-05-17T14:57:25.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 241302528 bytes
2024-05-17T14:57:25.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 269521920 bytes
2024-05-17T14:57:25.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 311182336 bytes
2024-05-17T14:57:25.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 381058048 bytes
2024-05-17T14:57:25.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 381666304 bytes
2024-05-17T14:57:25.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 443592704 bytes
2024-05-17T14:57:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 465134592 bytes
2024-05-17T14:57:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 508709888 bytes
2024-05-17T14:57:26.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 518027264 bytes
2024-05-17T14:57:26.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 521848832 bytes
2024-05-17T14:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 569368576 bytes
2024-05-17T14:57:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 574386176 bytes
2024-05-17T14:57:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 627085312 bytes
2024-05-17T14:57:26.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 627720192 bytes
2024-05-17T14:57:26.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 701791232 bytes
2024-05-17T14:57:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 764242944 bytes
2024-05-17T14:57:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 787232768 bytes
2024-05-17T14:57:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 801861632 bytes
2024-05-17T14:57:27.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 905465856 bytes
2024-05-17T14:57:27.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 926622720 bytes
2024-05-17T14:57:27.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 968330240 bytes
2024-05-17T14:57:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1024733184 bytes
2024-05-17T14:57:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1076964352 bytes
2024-05-17T14:57:28.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1145575424 bytes
2024-05-17T14:57:28.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1209230336 bytes
2024-05-17T14:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1222580224 bytes
2024-05-17T14:57:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1255453696 bytes
2024-05-17T14:57:29.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1338234880 bytes
2024-05-17T14:57:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1394560000 bytes
2024-05-17T14:57:29.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1441256448 bytes
2024-05-17T14:57:29.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1499767808 bytes
2024-05-17T14:57:30.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1556563968 bytes
2024-05-17T14:57:30.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1557163008 bytes
2024-05-17T14:57:30.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1572685824 bytes
2024-05-17T14:57:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1574525952 bytes
2024-05-17T14:57:30.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1697446912 bytes
2024-05-17T14:57:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1701443584 bytes
2024-05-17T14:57:30.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1788539904 bytes
2024-05-17T14:57:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1863534592 bytes
2024-05-17T14:57:31.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1885035520 bytes
2024-05-17T14:57:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1929935872 bytes
2024-05-17T14:57:31.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1954418688 bytes
2024-05-17T14:57:31.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2009546752 bytes
2024-05-17T14:57:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2101110784 bytes
2024-05-17T14:57:32.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2135849984 bytes
2024-05-17T14:57:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2211033088 bytes
2024-05-17T14:57:32.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2293167104 bytes
2024-05-17T14:57:32.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2341386240 bytes
2024-05-17T14:57:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2406617088 bytes
2024-05-17T14:57:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2416205824 bytes
2024-05-17T14:57:33.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2468859904 bytes
2024-05-17T14:57:33.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2487892992 bytes
2024-05-17T14:57:33.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2494298112 bytes
2024-05-17T14:57:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2574018560 bytes
2024-05-17T14:57:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2597415936 bytes
2024-05-17T14:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2610405376 bytes
2024-05-17T14:57:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2683450368 bytes
2024-05-17T14:57:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2731461632 bytes
2024-05-17T14:57:34.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2757757952 bytes
2024-05-17T14:57:34.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2758360064 bytes
2024-05-17T14:57:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2798557184 bytes
2024-05-17T14:57:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2842289152 bytes
2024-05-17T14:57:35.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2890715136 bytes
2024-05-17T14:57:35.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 2930779136 bytes
2024-05-17T14:57:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 2977620992 bytes
2024-05-17T14:57:35.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 3114543104 bytes
2024-05-17T14:57:35.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 3132755968 bytes
2024-05-17T14:57:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 3168111616 bytes
2024-05-17T14:57:36.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3219806208 bytes
2024-05-17T14:57:36.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7700 IOs, 3266710528 bytes
2024-05-17T14:57:36.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7800 IOs, 3347169280 bytes
2024-05-17T14:57:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7900 IOs, 3492368384 bytes
2024-05-17T14:57:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T14:57:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:57:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T14:57:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T14:57:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:37.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:57:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:57:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:57:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T14:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T14:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       down+unknown
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: status not found
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update:
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: down+unknown
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: status not found
2024-05-17T14:57:38.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: '
2024-05-17T14:57:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T14:57:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 1
2024-05-17T14:57:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 2
2024-05-17T14:57:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T14:57:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:57:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:57:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:57:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:57:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:57:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T14:57:40.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       down+unknown
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: status not found
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update:
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: down+unknown
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: status not found
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: '
2024-05-17T14:57:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T14:57:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 1
2024-05-17T14:57:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 4
2024-05-17T14:57:41.136 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi171.stderr:2024-05-17T14:57:41.130+0000 7f014dbd4880 -1 rbd::mirror::Mirror: 0x55ff01848120 update_pool_replayers: restarting failed pool replayer for uuid: b38475ef-ee9c-43e8-a5fc-5f468e3a0055 cluster: cluster2 client: client.mirror
2024-05-17T14:57:41.155 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi171.stderr:2024-05-17T14:57:41.150+0000 7f5dfdb50880 -1 rbd::mirror::Mirror: 0x55a97f9fa120 update_pool_replayers: restarting failed pool replayer for uuid: b38475ef-ee9c-43e8-a5fc-5f468e3a0055 cluster: cluster2 client: client.mirror
2024-05-17T14:57:41.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi171.stderr:2024-05-17T14:57:41.242+0000 7f014dbd4880 -1 rbd::mirror::Mirror: 0x55ff01848120 update_pool_replayers: restarting failed pool replayer for uuid: 3d680bc2-aeba-43c8-8bee-c93718c988fb cluster: cluster2 client: client.mirror
2024-05-17T14:57:41.256 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi171.stderr:2024-05-17T14:57:41.250+0000 7f5dfdb50880 -1 rbd::mirror::Mirror: 0x55a97f9fa120 update_pool_replayers: restarting failed pool replayer for uuid: 3d680bc2-aeba-43c8-8bee-c93718c988fb cluster: cluster2 client: client.mirror
2024-05-17T14:57:41.360 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi171.stderr:2024-05-17T14:57:41.354+0000 7f470d8d1880 -1 rbd::mirror::Mirror: 0x55692ee0e120 update_pool_replayers: restarting failed pool replayer for uuid: b38475ef-ee9c-43e8-a5fc-5f468e3a0055 cluster: cluster2 client: client.mirror
2024-05-17T14:57:41.421 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi171.stderr:2024-05-17T14:57:41.418+0000 7f470d8d1880 -1 rbd::mirror::Mirror: 0x55692ee0e120 update_pool_replayers: restarting failed pool replayer for uuid: 3d680bc2-aeba-43c8-8bee-c93718c988fb cluster: cluster2 client: client.mirror
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:57:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:57:44.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:57:44.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T14:57:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T14:57:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:57:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       down+unknown
2024-05-17T14:57:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: status not found
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update:
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: down+unknown
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: status not found
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: '
2024-05-17T14:57:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T14:57:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 1
2024-05-17T14:57:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:57:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 8
2024-05-17T14:57:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T14:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T14:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1715957834,"remote_snapshot_timestamp":1715957834,"replay_state":"idle"}
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.0 on smithi171
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 14:57:51
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 14:57:48'
2024-05-17T14:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T14:57:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T14:57:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T14:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1715957834,"remote_snapshot_timestamp":1715957834,"replay_state":"idle"}
2024-05-17T14:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T14:57:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T14:57:52.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T14:57:52.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T14:57:52.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.0 on smithi171
2024-05-17T14:57:52.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T14:57:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T14:57:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:57:51'
2024-05-17T14:57:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 14:57:51'
2024-05-17T14:57:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 14:57:51'
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:57:52.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T14:57:52.469 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4220</service_id><instance_id>4322</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4230</service_id><instance_id>4342</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4299</service_id><instance_id>4302</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:0.0,&quot;bytes_per_snapshot&quot;:0.0,&quot;last_snapshot_bytes&quot;:0,&quot;last_snapshot_sync_seconds&quot;:0,&quot;local_snapshot_timestamp&quot;:1715957834,&quot;remote_snapshot_timestamp&quot;:1715957834,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4299</service_id><instance_id>4302</instance_id><daemon_id>mirror.0</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 14:57:51</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 14:57:48</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T14:57:52.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T14:57:52.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T14:57:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:57:51'
2024-05-17T14:57:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T14:57:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T14:57:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T14:57:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T14:57:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T14:57:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:57:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap1
2024-05-17T14:57:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap1
2024-05-17T14:57:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:57:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:57:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:57:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1
2024-05-17T14:57:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20
2024-05-17T14:58:08.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T14:58:08.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:08.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:08.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:08.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:58:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:58:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T14:58:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T14:58:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T14:58:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:58:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T14:58:09.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:58:09.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:58:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:58:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:58:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:58:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:58:09.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:58:09.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:58:09.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:58:09.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:58:09.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:58:09.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:58:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:58:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:58:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:58:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:58:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:58:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:58:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:58:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:58:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:58:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:58:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:58:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:58:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:58:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:58:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:58:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T14:58:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T14:58:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T14:58:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T14:58:09.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T14:58:09.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T14:58:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T14:58:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T14:58:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T14:58:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:58:09.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T14:58:09.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T14:58:09.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T14:58:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T14:58:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:09.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:09.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T14:58:10.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 5
2024-05-17T14:58:10.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:58:10.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:10.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:58:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T14:58:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:58:10.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T14:58:10.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=5
2024-05-17T14:58:10.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:58:10.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:58:10.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T14:58:10.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:10.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:58:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:58:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:58:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=3
2024-05-17T14:58:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 5 = 3
2024-05-17T14:58:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:58:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T14:58:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:11.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:58:11.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:58:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:58:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:58:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T14:58:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:58:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T14:58:11.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T14:58:11.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 5 = ''
2024-05-17T14:58:11.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:58:11.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T14:58:12.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:12.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:12.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:12.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:12.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:12.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:58:12.134 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:58:12.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:58:12.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:58:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T14:58:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:58:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T14:58:12.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T14:58:12.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 5 = ''
2024-05-17T14:58:12.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:58:12.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T14:58:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:58:13.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:58:13.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:58:13.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:58:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=5
2024-05-17T14:58:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 5 = 5
2024-05-17T14:58:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap1
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T14:58:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:58:15.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap1
2024-05-17T14:58:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T14:58:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 14:57:23 2024
2024-05-17T14:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T14:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T14:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T14:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T14:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T14:58:15.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:15.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:58:15.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap1
2024-05-17T14:58:15.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T14:58:15.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T14:58:15.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:58:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T14:58:16.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:58:17.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap1-clean
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap1-clean
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1-clean
2024-05-17T14:58:17.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T14:58:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T14:58:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T14:58:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T14:58:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T14:58:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:58:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T14:58:17.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 618496 bytes
2024-05-17T14:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 49055744 bytes
2024-05-17T14:58:18.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 99549184 bytes
2024-05-17T14:58:18.213 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 126024704 bytes
2024-05-17T14:58:18.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 136610816 bytes
2024-05-17T14:58:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 144112640 bytes
2024-05-17T14:58:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 195347456 bytes
2024-05-17T14:58:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 270598144 bytes
2024-05-17T14:58:18.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 342431744 bytes
2024-05-17T14:58:19.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 380802048 bytes
2024-05-17T14:58:19.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 426796032 bytes
2024-05-17T14:58:19.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 440395776 bytes
2024-05-17T14:58:19.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 505991168 bytes
2024-05-17T14:58:19.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 613509120 bytes
2024-05-17T14:58:20.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 637763584 bytes
2024-05-17T14:58:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 679004160 bytes
2024-05-17T14:58:20.304 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 702997504 bytes
2024-05-17T14:58:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 779386880 bytes
2024-05-17T14:58:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 792508416 bytes
2024-05-17T14:58:20.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 843161600 bytes
2024-05-17T14:58:21.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 890801152 bytes
2024-05-17T14:58:21.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 918492160 bytes
2024-05-17T14:58:21.207 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 919208960 bytes
2024-05-17T14:58:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1012856832 bytes
2024-05-17T14:58:21.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1064869888 bytes
2024-05-17T14:58:21.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1077577728 bytes
2024-05-17T14:58:21.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1135314944 bytes
2024-05-17T14:58:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1160518656 bytes
2024-05-17T14:58:22.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1185272832 bytes
2024-05-17T14:58:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1249546240 bytes
2024-05-17T14:58:22.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1274545152 bytes
2024-05-17T14:58:22.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1340338176 bytes
2024-05-17T14:58:22.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1370364928 bytes
2024-05-17T14:58:22.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1395744768 bytes
2024-05-17T14:58:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1419028480 bytes
2024-05-17T14:58:23.158 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1427380224 bytes
2024-05-17T14:58:23.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1502789632 bytes
2024-05-17T14:58:23.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1537801216 bytes
2024-05-17T14:58:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1557070848 bytes
2024-05-17T14:58:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1598558208 bytes
2024-05-17T14:58:23.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1629893632 bytes
2024-05-17T14:58:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1649831936 bytes
2024-05-17T14:58:24.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1698280448 bytes
2024-05-17T14:58:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1772096512 bytes
2024-05-17T14:58:24.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1816307712 bytes
2024-05-17T14:58:24.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1825225728 bytes
2024-05-17T14:58:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1926142976 bytes
2024-05-17T14:58:25.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1962486784 bytes
2024-05-17T14:58:25.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2001820672 bytes
2024-05-17T14:58:25.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2002428928 bytes
2024-05-17T14:58:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2010889216 bytes
2024-05-17T14:58:25.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2057203712 bytes
2024-05-17T14:58:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2061697024 bytes
2024-05-17T14:58:25.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2096095232 bytes
2024-05-17T14:58:25.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2151697408 bytes
2024-05-17T14:58:26.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2167729152 bytes
2024-05-17T14:58:26.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2245085184 bytes
2024-05-17T14:58:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2270654464 bytes
2024-05-17T14:58:26.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2322279424 bytes
2024-05-17T14:58:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2395352064 bytes
2024-05-17T14:58:26.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2416971776 bytes
2024-05-17T14:58:27.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2456613888 bytes
2024-05-17T14:58:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2457227264 bytes
2024-05-17T14:58:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2521743360 bytes
2024-05-17T14:58:27.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2542491648 bytes
2024-05-17T14:58:27.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2560329728 bytes
2024-05-17T14:58:27.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2596169728 bytes
2024-05-17T14:58:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2619365376 bytes
2024-05-17T14:58:27.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2670595072 bytes
2024-05-17T14:58:28.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2674387968 bytes
2024-05-17T14:58:28.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 2750952448 bytes
2024-05-17T14:58:28.384 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 2771911680 bytes
2024-05-17T14:58:28.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 2797918208 bytes
2024-05-17T14:58:28.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 2888493056 bytes
2024-05-17T14:58:28.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 2920392704 bytes
2024-05-17T14:58:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3016016896 bytes
2024-05-17T14:58:29.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7700 IOs, 3048620032 bytes
2024-05-17T14:58:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7800 IOs, 3070142464 bytes
2024-05-17T14:58:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7900 IOs, 3084199936 bytes
2024-05-17T14:58:29.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8000 IOs, 3127236608 bytes
2024-05-17T14:58:29.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8100 IOs, 3145299968 bytes
2024-05-17T14:58:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8200 IOs, 3168102400 bytes
2024-05-17T14:58:30.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8300 IOs, 3204950016 bytes
2024-05-17T14:58:30.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8400 IOs, 3253234688 bytes
2024-05-17T14:58:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8500 IOs, 3253847040 bytes
2024-05-17T14:58:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8600 IOs, 3285243904 bytes
2024-05-17T14:58:30.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8700 IOs, 3324811264 bytes
2024-05-17T14:58:30.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8800 IOs, 3337437184 bytes
2024-05-17T14:58:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:8900 IOs, 3377702912 bytes
2024-05-17T14:58:30.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:9000 IOs, 3398128640 bytes
2024-05-17T14:58:31.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:9100 IOs, 3458825216 bytes
2024-05-17T14:58:31.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:9200 IOs, 3484823552 bytes
2024-05-17T14:58:31.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:9300 IOs, 3542759424 bytes
2024-05-17T14:58:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T14:58:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:58:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T14:58:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:58:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:58:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:58:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:58:32.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:58:32.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T14:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T14:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T14:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":17669734.4,"bytes_per_snapshot":265046016.0,"last_snapshot_bytes":530092032,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957890,"remote_snapshot_timestamp":1715957890,"replay_state":"idle"}
2024-05-17T14:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.0 on smithi171
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 14:58:21
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 14:58:18'
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T14:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T14:58:32.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T14:58:32.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":17669734.4,"bytes_per_snapshot":265046016.0,"last_snapshot_bytes":530092032,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957890,"remote_snapshot_timestamp":1715957890,"replay_state":"idle"}
2024-05-17T14:58:32.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:58:32.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T14:58:32.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T14:58:32.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T14:58:32.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T14:58:32.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.0 on smithi171
2024-05-17T14:58:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T14:58:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:58:21'
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 14:58:21'
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 14:58:21'
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T14:58:32.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4220</service_id><instance_id>4322</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4230</service_id><instance_id>4342</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4299</service_id><instance_id>4302</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:17669734.4,&quot;bytes_per_snapshot&quot;:265046016.0,&quot;last_snapshot_bytes&quot;:530092032,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715957890,&quot;remote_snapshot_timestamp&quot;:1715957890,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4299</service_id><instance_id>4302</instance_id><daemon_id>mirror.0</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 14:58:21</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 14:58:18</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T14:58:32.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T14:58:32.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T14:58:32.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:58:21'
2024-05-17T14:58:32.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T14:58:32.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T14:58:32.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T14:58:32.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T14:58:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap2
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap2
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:58:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2
2024-05-17T14:58:32.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20
2024-05-17T14:58:34.989 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0
2024-05-17T14:58:34.990 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15
2024-05-17T14:58:34.990 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons
2024-05-17T14:58:34.991 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T14:58:34.985+0000 7f9b4a32c640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0  (PID: 29317) UID: 1000
2024-05-17T14:58:41.991 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0
2024-05-17T14:58:41.991 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T14:58:41.992 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0
2024-05-17T14:58:41.992 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon
2024-05-17T14:58:41.992 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0
2024-05-17T14:58:41.995 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started
2024-05-17T14:58:41.995 INFO:tasks.rbd_mirror_thrash:waiting for 80 secs before thrashing
2024-05-17T14:58:47.987 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9129768786443766) > weight (0.25)
2024-05-17T14:58:47.987 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1
2024-05-17T14:58:47.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15
2024-05-17T14:58:47.988 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons
2024-05-17T14:58:47.989 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi171.stderr:2024-05-17T14:58:47.985+0000 7f470a88c640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1  (PID: 29309) UID: 1000
2024-05-17T14:58:52.989 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1
2024-05-17T14:58:52.989 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T14:58:52.989 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1
2024-05-17T14:58:52.989 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon
2024-05-17T14:58:52.990 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1
2024-05-17T14:58:52.993 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started
2024-05-17T14:58:52.993 INFO:tasks.rbd_mirror_thrash:waiting for 105 secs before thrashing
2024-05-17T14:59:03.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:59:04.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T14:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T14:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T14:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:59:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T14:59:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:59:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:59:04.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:59:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:59:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:59:04.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:59:04.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:59:04.263 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T14:59:04.263 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T14:59:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T14:59:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T14:59:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T14:59:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T14:59:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T14:59:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T14:59:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T14:59:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T14:59:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:59:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T14:59:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T14:59:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T14:59:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T14:59:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T14:59:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 8
2024-05-17T14:59:06.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:59:06.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:06.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:06.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:06.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:59:06.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T14:59:06.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T14:59:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=8
2024-05-17T14:59:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:59:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:59:06.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T14:59:06.372 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:59:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:06.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:59:06.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:59:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T14:59:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:59:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T14:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T14:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 8 = ''
2024-05-17T14:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T14:59:06.844 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:59:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:59:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:59:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T14:59:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:59:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T14:59:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T14:59:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 8 = ''
2024-05-17T14:59:06.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:59:06.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T14:59:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:59:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:07.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:59:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:59:07.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T14:59:07.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:59:07.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T14:59:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T14:59:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 8 = ''
2024-05-17T14:59:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:59:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T14:59:09.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:59:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:09.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:59:09.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=8
2024-05-17T14:59:09.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 8 = 8
2024-05-17T14:59:09.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:09.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap2
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T14:59:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:59:10.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap2
2024-05-17T14:59:10.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T14:59:10.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 4
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 14:57:23 2024
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T14:59:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T14:59:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T14:59:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:59:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap1-clean ']'
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap1-clean
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1-clean
2024-05-17T14:59:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T14:59:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T14:59:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T14:59:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T14:59:11.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T14:59:12.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:12.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap2
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T14:59:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T14:59:13.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T14:59:13.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:14.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T14:59:14.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T14:59:14.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:14.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap2-clean
2024-05-17T14:59:14.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap2-clean
2024-05-17T14:59:14.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:14.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:14.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:14.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2-clean
2024-05-17T14:59:14.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T14:59:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T14:59:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T14:59:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T14:59:15.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=12
2024-05-17T14:59:15.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T14:59:15.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T14:59:15.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 57841664 bytes
2024-05-17T14:59:15.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 83213312 bytes
2024-05-17T14:59:15.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 116917248 bytes
2024-05-17T14:59:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 140470272 bytes
2024-05-17T14:59:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 215313408 bytes
2024-05-17T14:59:16.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 237887488 bytes
2024-05-17T14:59:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 302143488 bytes
2024-05-17T14:59:16.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 325246976 bytes
2024-05-17T14:59:16.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 362491904 bytes
2024-05-17T14:59:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 404790272 bytes
2024-05-17T14:59:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 405396480 bytes
2024-05-17T14:59:17.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 455065600 bytes
2024-05-17T14:59:17.152 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 484959232 bytes
2024-05-17T14:59:17.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 618756096 bytes
2024-05-17T14:59:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 665755648 bytes
2024-05-17T14:59:17.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 678051840 bytes
2024-05-17T14:59:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 715931648 bytes
2024-05-17T14:59:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 771847168 bytes
2024-05-17T14:59:18.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 800513024 bytes
2024-05-17T14:59:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 867352576 bytes
2024-05-17T14:59:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 891394048 bytes
2024-05-17T14:59:18.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 915127296 bytes
2024-05-17T14:59:18.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 961247232 bytes
2024-05-17T14:59:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 967143424 bytes
2024-05-17T14:59:19.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1007357952 bytes
2024-05-17T14:59:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1054003200 bytes
2024-05-17T14:59:19.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1143273472 bytes
2024-05-17T14:59:19.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1163452416 bytes
2024-05-17T14:59:19.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1252773888 bytes
2024-05-17T14:59:20.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1268586496 bytes
2024-05-17T14:59:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1297251328 bytes
2024-05-17T14:59:20.245 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1306183680 bytes
2024-05-17T14:59:20.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1312977920 bytes
2024-05-17T14:59:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1366427648 bytes
2024-05-17T14:59:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1399788544 bytes
2024-05-17T14:59:20.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1446416384 bytes
2024-05-17T14:59:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1496111104 bytes
2024-05-17T14:59:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1535693824 bytes
2024-05-17T14:59:21.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1547158528 bytes
2024-05-17T14:59:21.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1563355136 bytes
2024-05-17T14:59:21.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1614181376 bytes
2024-05-17T14:59:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1680483328 bytes
2024-05-17T14:59:21.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1729053696 bytes
2024-05-17T14:59:22.035 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1765504000 bytes
2024-05-17T14:59:22.234 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1824559104 bytes
2024-05-17T14:59:22.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1862969344 bytes
2024-05-17T14:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1888055296 bytes
2024-05-17T14:59:22.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1943563264 bytes
2024-05-17T14:59:22.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1983261696 bytes
2024-05-17T14:59:23.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2024095744 bytes
2024-05-17T14:59:23.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2043815936 bytes
2024-05-17T14:59:23.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2119032832 bytes
2024-05-17T14:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2207375360 bytes
2024-05-17T14:59:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2234431488 bytes
2024-05-17T14:59:23.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2277335040 bytes
2024-05-17T14:59:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2351209472 bytes
2024-05-17T14:59:24.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2380712960 bytes
2024-05-17T14:59:24.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2438137856 bytes
2024-05-17T14:59:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2451444736 bytes
2024-05-17T14:59:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2511689728 bytes
2024-05-17T14:59:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2595932160 bytes
2024-05-17T14:59:25.317 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2679273472 bytes
2024-05-17T14:59:25.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2728717312 bytes
2024-05-17T14:59:25.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2800101376 bytes
2024-05-17T14:59:25.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2851918848 bytes
2024-05-17T14:59:26.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2876337152 bytes
2024-05-17T14:59:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2974355456 bytes
2024-05-17T14:59:26.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2974962688 bytes
2024-05-17T14:59:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2986883072 bytes
2024-05-17T14:59:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 3045138432 bytes
2024-05-17T14:59:26.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3072369664 bytes
2024-05-17T14:59:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3102840832 bytes
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:59:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:59:27.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:59:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:59:28.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T14:59:28.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":17874875.73,"bytes_per_snapshot":533169152.0,"last_snapshot_bytes":536246272,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957945,"remote_snapshot_timestamp":1715957945,"replay_state":"idle"}
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.0 on smithi171
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 14:59:21
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T14:59:28.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 14:59:18'
2024-05-17T14:59:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T14:59:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T14:59:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T14:59:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":17874875.73,"bytes_per_snapshot":533169152.0,"last_snapshot_bytes":536246272,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957945,"remote_snapshot_timestamp":1715957945,"replay_state":"idle"}
2024-05-17T14:59:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T14:59:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T14:59:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T14:59:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T14:59:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T14:59:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.0 on smithi171
2024-05-17T14:59:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T14:59:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T14:59:28.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:59:21'
2024-05-17T14:59:28.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 14:59:21'
2024-05-17T14:59:28.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 14:59:21'
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T14:59:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T14:59:28.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4220</service_id><instance_id>4322</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4299</service_id><instance_id>4302</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:17874875.73,&quot;bytes_per_snapshot&quot;:533169152.0,&quot;last_snapshot_bytes&quot;:536246272,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715957945,&quot;remote_snapshot_timestamp&quot;:1715957945,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4299</service_id><instance_id>4302</instance_id><daemon_id>mirror.0</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 14:59:21</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 14:59:18</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T14:59:28.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T14:59:28.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T14:59:28.284 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:59:21'
2024-05-17T14:59:28.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T14:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T14:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T14:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T14:59:28.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T14:59:28.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap3
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap3
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3
2024-05-17T14:59:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20
2024-05-17T14:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T14:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T14:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T14:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T14:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T14:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T14:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T14:59:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T14:59:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T14:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T14:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T14:59:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T14:59:57.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:59:57.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:59:57.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T14:59:57.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T14:59:57.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T14:59:57.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T14:59:57.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T14:59:57.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T14:59:57.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T14:59:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:57.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:57.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:57.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T14:59:57.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T14:59:57.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T14:59:57.372 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:59:57.372 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T14:59:57.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T14:59:57.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T14:59:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T14:59:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T14:59:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 11
2024-05-17T14:59:59.263 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:59:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T14:59:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T14:59:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T14:59:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:59.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T14:59:59.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=11
2024-05-17T14:59:59.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:59:59.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:59:59.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T14:59:59.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:59.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T14:59:59.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T14:59:59.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T14:59:59.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T14:59:59.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T14:59:59.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T14:59:59.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T14:59:59.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T14:59:59.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T14:59:59.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T14:59:59.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T14:59:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T14:59:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 11 = ''
2024-05-17T14:59:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T14:59:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:00:00.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:00.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:00.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:00.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:00.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:00.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:00.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:00:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:00:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:00:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:00:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:00:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 11 = ''
2024-05-17T15:00:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:00:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:00:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:00.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:00:00.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:00:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:00:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:00:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:00:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 11 = ''
2024-05-17T15:00:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:00:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:00:01.996 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9983479459358696) > weight (0.25)
2024-05-17T15:00:01.997 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.96343665172399) > weight (0.25)
2024-05-17T15:00:01.997 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4685150451711402) > weight (0.25)
2024-05-17T15:00:01.997 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.45156055782822413) > weight (0.25)
2024-05-17T15:00:01.997 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing
2024-05-17T15:00:02.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:02.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:02.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:02.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=11
2024-05-17T15:00:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 11 = 11
2024-05-17T15:00:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap3
2024-05-17T15:00:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3
2024-05-17T15:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:00:03.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap3
2024-05-17T15:00:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:00:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:00:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 6
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 14:59:11 2024
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:00:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:00:03.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:00:03.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:00:03.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap2-clean ']'
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap2-clean
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2-clean
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:00:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:04.424 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:05.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:05.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:05.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap3
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:06.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:07.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:08.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap3-clean
2024-05-17T15:00:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap3-clean
2024-05-17T15:00:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3-clean
2024-05-17T15:00:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:00:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=8
2024-05-17T15:00:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:00:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:00:09.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 16182272 bytes
2024-05-17T15:00:09.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 30050304 bytes
2024-05-17T15:00:09.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 45794304 bytes
2024-05-17T15:00:09.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 58760192 bytes
2024-05-17T15:00:10.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 120714240 bytes
2024-05-17T15:00:10.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 163637248 bytes
2024-05-17T15:00:10.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 251788288 bytes
2024-05-17T15:00:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 296719360 bytes
2024-05-17T15:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 382863360 bytes
2024-05-17T15:00:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 429903872 bytes
2024-05-17T15:00:11.209 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 440631296 bytes
2024-05-17T15:00:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 476303360 bytes
2024-05-17T15:00:11.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 531053568 bytes
2024-05-17T15:00:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 601425920 bytes
2024-05-17T15:00:11.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 664476672 bytes
2024-05-17T15:00:12.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 680341504 bytes
2024-05-17T15:00:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 754006016 bytes
2024-05-17T15:00:12.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 767028224 bytes
2024-05-17T15:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 796391424 bytes
2024-05-17T15:00:12.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 836331520 bytes
2024-05-17T15:00:12.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 895016960 bytes
2024-05-17T15:00:12.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 915219456 bytes
2024-05-17T15:00:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 931312640 bytes
2024-05-17T15:00:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 947619840 bytes
2024-05-17T15:00:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1008742400 bytes
2024-05-17T15:00:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1062020096 bytes
2024-05-17T15:00:13.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1123020800 bytes
2024-05-17T15:00:13.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1165850624 bytes
2024-05-17T15:00:14.149 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1212460032 bytes
2024-05-17T15:00:14.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1267385344 bytes
2024-05-17T15:00:14.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1321358336 bytes
2024-05-17T15:00:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1409287168 bytes
2024-05-17T15:00:14.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1445259264 bytes
2024-05-17T15:00:15.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1482467328 bytes
2024-05-17T15:00:15.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1496028160 bytes
2024-05-17T15:00:15.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1536258048 bytes
2024-05-17T15:00:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1567603712 bytes
2024-05-17T15:00:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1633421312 bytes
2024-05-17T15:00:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1670491136 bytes
2024-05-17T15:00:16.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1715872768 bytes
2024-05-17T15:00:16.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1753395200 bytes
2024-05-17T15:00:16.384 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1786573824 bytes
2024-05-17T15:00:16.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1787188224 bytes
2024-05-17T15:00:16.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1884483584 bytes
2024-05-17T15:00:16.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1913230336 bytes
2024-05-17T15:00:17.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1960197120 bytes
2024-05-17T15:00:17.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1972060160 bytes
2024-05-17T15:00:17.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2007618560 bytes
2024-05-17T15:00:17.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:00:17.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:00:17.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:00:17.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:17.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:00:17.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:00:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:00:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533169152.0,"last_snapshot_bytes":536246272,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957945,"remote_snapshot_timestamp":1715957945,"replay_state":"idle"}
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.0 on smithi171
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 14:59:51
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:00:18'
2024-05-17T15:00:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:00:18.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:00:18.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:00:18.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533169152.0,"last_snapshot_bytes":536246272,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957945,"remote_snapshot_timestamp":1715957945,"replay_state":"idle"}
2024-05-17T15:00:18.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:00:18.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:00:18.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:00:18.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:00:18.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:00:18.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.0 on smithi171
2024-05-17T15:00:18.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:00:18.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:59:51'
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 14:59:51'
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 14:59:51'
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:00:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:00:18.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4220</service_id><instance_id>4322</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4299</service_id><instance_id>4302</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:0.0,&quot;bytes_per_snapshot&quot;:533169152.0,&quot;last_snapshot_bytes&quot;:536246272,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715957945,&quot;remote_snapshot_timestamp&quot;:1715957945,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4299</service_id><instance_id>4302</instance_id><daemon_id>mirror.0</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 14:59:51</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:00:18</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:00:18.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:00:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 14:59:51'
2024-05-17T15:00:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:00:18.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:00:18.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:00:18.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:00:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:00:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap4
2024-05-17T15:00:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap4
2024-05-17T15:00:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4
2024-05-17T15:00:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:00:37.994 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5970197779437194) > weight (0.25)
2024-05-17T15:00:37.994 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7353768840844561) > weight (0.25)
2024-05-17T15:00:37.994 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2
2024-05-17T15:00:37.995 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15
2024-05-17T15:00:37.995 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons
2024-05-17T15:00:37.996 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi171.stderr:2024-05-17T15:00:37.996+0000 7f014ab8f640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2  (PID: 29311) UID: 1000
2024-05-17T15:00:44.996 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2
2024-05-17T15:00:44.997 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:00:44.997 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2
2024-05-17T15:00:44.997 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon
2024-05-17T15:00:44.997 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2
2024-05-17T15:00:45.001 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started
2024-05-17T15:00:45.001 INFO:tasks.rbd_mirror_thrash:waiting for 78 secs before thrashing
2024-05-17T15:00:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:00:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:00:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:00:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:00:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:00:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:00:47.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:00:47.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:47.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:47.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:47.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:00:47.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:00:47.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:00:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:00:47.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:00:47.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:00:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:00:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:00:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:00:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:00:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:00:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:00:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:00:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:00:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:00:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:00:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:00:47.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:00:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:00:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:00:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:00:47.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:00:47.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:00:47.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:00:47.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:47.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:00:47.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:00:47.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:00:47.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:00:47.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:00:47.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:00:47.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:00:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:00:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:00:49.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 14
2024-05-17T15:00:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:00:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:00:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:00:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:00:49.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=14
2024-05-17T15:00:49.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:00:49.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:00:49.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:00:49.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:49.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:00:49.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:00:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:00:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:00:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:00:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 14 = ''
2024-05-17T15:00:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:00:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:00:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:50.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:00:50.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:50.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:00:50.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:00:50.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:00:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:00:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 14 = ''
2024-05-17T15:00:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:00:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:00:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:51.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:51.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:51.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:51.335 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:00:51.335 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:00:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:00:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:00:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:00:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 14 = ''
2024-05-17T15:00:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:00:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:00:52.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:52.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:52.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:52.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:52.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:00:52.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:00:52.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:00:53.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=14
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 14 = 14
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap4
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:00:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:00:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:00:54.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap4
2024-05-17T15:00:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:00:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:00:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:00:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 8
2024-05-17T15:00:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:00:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 14:59:11 2024
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:00:54.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap3-clean ']'
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap3-clean
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3-clean
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:54.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:55.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:55.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap4
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:00:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:57.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:58.325 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:00:58.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:00:58.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:00:58.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap4-clean
2024-05-17T15:00:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap4-clean
2024-05-17T15:00:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4-clean
2024-05-17T15:00:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=8
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:00:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 20783104 bytes
2024-05-17T15:00:59.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 68436992 bytes
2024-05-17T15:00:59.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 131689472 bytes
2024-05-17T15:01:00.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 183359488 bytes
2024-05-17T15:01:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 242105344 bytes
2024-05-17T15:01:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 276887552 bytes
2024-05-17T15:01:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 382590976 bytes
2024-05-17T15:01:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 450486272 bytes
2024-05-17T15:01:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 458086400 bytes
2024-05-17T15:01:01.209 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 472315904 bytes
2024-05-17T15:01:01.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 511481856 bytes
2024-05-17T15:01:01.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 575029248 bytes
2024-05-17T15:01:01.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 632233984 bytes
2024-05-17T15:01:01.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 664402944 bytes
2024-05-17T15:01:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 720296960 bytes
2024-05-17T15:01:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 741422080 bytes
2024-05-17T15:01:02.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 799483904 bytes
2024-05-17T15:01:02.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 840725504 bytes
2024-05-17T15:01:02.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 906592256 bytes
2024-05-17T15:01:02.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 931434496 bytes
2024-05-17T15:01:02.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 935620608 bytes
2024-05-17T15:01:03.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 992409600 bytes
2024-05-17T15:01:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1069278208 bytes
2024-05-17T15:01:03.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1152884736 bytes
2024-05-17T15:01:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1164740608 bytes
2024-05-17T15:01:03.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1231643648 bytes
2024-05-17T15:01:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1339695104 bytes
2024-05-17T15:01:04.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1361673216 bytes
2024-05-17T15:01:04.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1394217984 bytes
2024-05-17T15:01:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1414061056 bytes
2024-05-17T15:01:04.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1496801280 bytes
2024-05-17T15:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1549561856 bytes
2024-05-17T15:01:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1555223552 bytes
2024-05-17T15:01:05.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1609937920 bytes
2024-05-17T15:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1672645632 bytes
2024-05-17T15:01:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1701830656 bytes
2024-05-17T15:01:05.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1777406976 bytes
2024-05-17T15:01:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1829873664 bytes
2024-05-17T15:01:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1848286208 bytes
2024-05-17T15:01:06.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1883350016 bytes
2024-05-17T15:01:06.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1927872512 bytes
2024-05-17T15:01:06.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1965671424 bytes
2024-05-17T15:01:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1975420928 bytes
2024-05-17T15:01:06.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 2020067328 bytes
2024-05-17T15:01:06.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 2020668416 bytes
2024-05-17T15:01:07.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2041713664 bytes
2024-05-17T15:01:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2102604800 bytes
2024-05-17T15:01:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:01:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:01:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:01:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:01:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:01:08.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:01:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:01:08.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":534043136.0,"last_snapshot_bytes":531840000,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957998,"remote_snapshot_timestamp":1715957998,"replay_state":"idle"}
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.0 on smithi171
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:00:51
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:00:48'
2024-05-17T15:01:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:01:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:01:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:01:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":534043136.0,"last_snapshot_bytes":531840000,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715957998,"remote_snapshot_timestamp":1715957998,"replay_state":"idle"}
2024-05-17T15:01:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:01:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:01:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:01:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:01:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:01:08.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.0 on smithi171
2024-05-17T15:01:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:01:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:00:51'
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 15:00:51'
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 15:00:51'
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:01:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:01:08.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4299</service_id><instance_id>4302</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4449</service_id><instance_id>4451</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:0.0,&quot;bytes_per_snapshot&quot;:534043136.0,&quot;last_snapshot_bytes&quot;:531840000,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715957998,&quot;remote_snapshot_timestamp&quot;:1715957998,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4299</service_id><instance_id>4302</instance_id><daemon_id>mirror.0</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 15:00:51</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:00:48</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:01:09.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:00:51'
2024-05-17T15:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap5
2024-05-17T15:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap5
2024-05-17T15:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5
2024-05-17T15:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:01:39.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:01:39.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:39.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:39.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:39.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:01:39.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:01:39.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:01:39.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:01:39.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:01:39.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:01:39.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:01:39.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:01:39.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:01:39.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:01:39.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:01:39.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:01:39.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:01:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:01:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:01:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:01:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:01:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:01:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:01:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:01:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:01:41.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 17
2024-05-17T15:01:41.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:01:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:01:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:01:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:01:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:01:41.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=17
2024-05-17T15:01:41.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:01:41.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:01:41.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:01:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:01:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:01:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:01:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 17 = ''
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:01:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:01:41.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.30513302408652976) > weight (0.25)
2024-05-17T15:01:41.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8801198131428112) > weight (0.25)
2024-05-17T15:01:41.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.37718061805352743) > weight (0.25)
2024-05-17T15:01:41.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8773455812270387) > weight (0.25)
2024-05-17T15:01:41.998 INFO:tasks.rbd_mirror_thrash:waiting for 74 secs before thrashing
2024-05-17T15:01:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:01:42.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:01:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:01:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:01:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:01:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:01:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:01:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:01:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 17 = ''
2024-05-17T15:01:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:01:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:01:43.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:43.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:43.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:43.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:43.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:43.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:01:43.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:01:43.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:01:43.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:01:43.245 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:01:43.245 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:01:43.245 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:01:43.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:01:43.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 17 = ''
2024-05-17T15:01:43.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:01:43.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:01:44.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:44.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:44.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:44.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:44.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:01:44.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:01:44.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:01:44.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=17
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 17 = 17
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap5
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:01:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:01:44.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:01:45.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap5
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 10
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:01:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:00:55 2024
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:01:45.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap4-clean ']'
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap4-clean
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4-clean
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:01:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:01:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:01:46.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:01:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:47.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:01:47.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:48.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:48.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap5
2024-05-17T15:01:48.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:48.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:01:48.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:50.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:01:50.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:50.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:01:50.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:01:50.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap5-clean
2024-05-17T15:01:50.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap5-clean
2024-05-17T15:01:50.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:50.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:50.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:50.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5-clean
2024-05-17T15:01:50.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:01:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:01:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:01:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:01:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:01:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:01:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:01:51.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:01:51.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:01:51.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:01:51.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 74531840 bytes
2024-05-17T15:01:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 112210944 bytes
2024-05-17T15:01:52.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 160496640 bytes
2024-05-17T15:01:52.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 161104896 bytes
2024-05-17T15:01:52.399 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 198025216 bytes
2024-05-17T15:01:52.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 238585856 bytes
2024-05-17T15:01:52.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 275997696 bytes
2024-05-17T15:01:52.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 340406272 bytes
2024-05-17T15:01:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 380701696 bytes
2024-05-17T15:01:53.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 405581824 bytes
2024-05-17T15:01:53.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 469712896 bytes
2024-05-17T15:01:53.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 548511744 bytes
2024-05-17T15:01:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 592832512 bytes
2024-05-17T15:01:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 609982464 bytes
2024-05-17T15:01:54.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 677292032 bytes
2024-05-17T15:01:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 725813248 bytes
2024-05-17T15:01:54.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 746474496 bytes
2024-05-17T15:01:54.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 754621440 bytes
2024-05-17T15:01:54.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 835069952 bytes
2024-05-17T15:01:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 870677504 bytes
2024-05-17T15:01:55.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 965778432 bytes
2024-05-17T15:01:55.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1003510784 bytes
2024-05-17T15:01:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1047635968 bytes
2024-05-17T15:01:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1112446976 bytes
2024-05-17T15:01:55.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1129665536 bytes
2024-05-17T15:01:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1175069696 bytes
2024-05-17T15:01:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1236045824 bytes
2024-05-17T15:01:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1244875776 bytes
2024-05-17T15:01:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1303830528 bytes
2024-05-17T15:01:56.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1351190528 bytes
2024-05-17T15:01:56.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1388704768 bytes
2024-05-17T15:01:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1414736896 bytes
2024-05-17T15:01:57.180 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1477417984 bytes
2024-05-17T15:01:57.318 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1495571456 bytes
2024-05-17T15:01:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1546817536 bytes
2024-05-17T15:01:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1560139776 bytes
2024-05-17T15:01:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1594035200 bytes
2024-05-17T15:01:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1638153216 bytes
2024-05-17T15:01:58.135 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1724686336 bytes
2024-05-17T15:01:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1759021056 bytes
2024-05-17T15:01:58.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1849633792 bytes
2024-05-17T15:01:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1890239488 bytes
2024-05-17T15:01:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1931819008 bytes
2024-05-17T15:01:59.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 2010772480 bytes
2024-05-17T15:01:59.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 2011392000 bytes
2024-05-17T15:01:59.312 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2028600320 bytes
2024-05-17T15:01:59.478 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2095352832 bytes
2024-05-17T15:01:59.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2097419264 bytes
2024-05-17T15:01:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2103241728 bytes
2024-05-17T15:01:59.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2163435520 bytes
2024-05-17T15:02:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2272535552 bytes
2024-05-17T15:02:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2321776640 bytes
2024-05-17T15:02:00.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2366487552 bytes
2024-05-17T15:02:00.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2414852096 bytes
2024-05-17T15:02:00.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2465030144 bytes
2024-05-17T15:02:01.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2508645376 bytes
2024-05-17T15:02:01.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2611652608 bytes
2024-05-17T15:02:01.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2647233536 bytes
2024-05-17T15:02:01.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2698051584 bytes
2024-05-17T15:02:01.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2743280640 bytes
2024-05-17T15:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2788816896 bytes
2024-05-17T15:02:02.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2829729792 bytes
2024-05-17T15:02:02.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2889976832 bytes
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:02:03.002 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0
2024-05-17T15:02:03.002 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15
2024-05-17T15:02:03.003 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons
2024-05-17T15:02:03.004 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T15:02:03.002+0000 7ffb5cd9d640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 30506) UID: 1000
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:02:03.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:02:03.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:02:03.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:02:03.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       down+stopped
2024-05-17T15:02:03.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: stopped
2024-05-17T15:02:03.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:02:03
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:01:48'
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:02:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 1
2024-05-17T15:02:03.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:02:03.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 2
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:02:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:02:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:02:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:02:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:02:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       down+stopped
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: stopped
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:02:03
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:01:48'
2024-05-17T15:02:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:02:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 1
2024-05-17T15:02:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:02:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 4
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:02:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:02:09.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:02:09.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:02:09.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:02:09.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1715958100,"remote_snapshot_timestamp":1715958100,"replay_state":"idle"}
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.3 on smithi171
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:02:07
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:01:48'
2024-05-17T15:02:09.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:02:09.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:02:09.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:02:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1715958100,"remote_snapshot_timestamp":1715958100,"replay_state":"idle"}
2024-05-17T15:02:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:02:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:02:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:02:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:02:09.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:02:09.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.3 on smithi171
2024-05-17T15:02:09.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:02:09.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:02:07'
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 15:02:07'
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 15:02:07'
2024-05-17T15:02:09.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:02:09.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:02:09.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:02:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4449</service_id><instance_id>4451</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:0.0,&quot;bytes_per_snapshot&quot;:0.0,&quot;last_snapshot_bytes&quot;:0,&quot;last_snapshot_sync_seconds&quot;:0,&quot;local_snapshot_timestamp&quot;:1715958100,&quot;remote_snapshot_timestamp&quot;:1715958100,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4228</service_id><instance_id>4325</instance_id><daemon_id>mirror.3</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 15:02:07</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:01:48</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:02:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:02:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:02:09.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:02:07'
2024-05-17T15:02:09.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:02:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:02:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:02:09.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:02:09.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:02:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap6
2024-05-17T15:02:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap6
2024-05-17T15:02:09.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:09.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:09.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:09.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6
2024-05-17T15:02:09.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:02:10.004 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0
2024-05-17T15:02:10.004 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:02:10.004 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0
2024-05-17T15:02:10.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon
2024-05-17T15:02:10.005 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0
2024-05-17T15:02:10.008 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started
2024-05-17T15:02:10.008 INFO:tasks.rbd_mirror_thrash:waiting for 110 secs before thrashing
2024-05-17T15:02:33.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:02:33.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:33.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:02:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:02:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:02:34.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:02:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:02:34.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:02:34.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:02:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:02:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:02:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:02:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:02:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:02:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:02:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:02:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:02:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:02:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:02:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:02:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:02:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:02:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:02:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:02:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:02:34.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:02:34.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:02:34.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:02:34.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:02:34.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:02:34.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:02:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:02:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:02:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:02:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T15:02:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:02:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:02:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T15:02:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T15:02:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:02:34.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T15:02:34.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T15:02:34.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T15:02:34.849 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:02:34.849 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:02:34.849 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:02:34.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T15:02:34.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:02:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:02:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T15:02:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T15:02:34.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:02:34.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T15:02:34.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:02:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T15:02:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T15:02:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:02:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T15:02:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T15:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:02:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:02:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T15:02:35.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:02:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:02:35.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:02:35.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:02:35.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:02:35.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:35.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:35.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:02:35.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:02:35.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:02:35.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:02:35.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:02:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:02:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:02:35.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:35.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:02:35.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:35.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:35.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:35.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:02:36.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 20
2024-05-17T15:02:36.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:02:36.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:36.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:36.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:36.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:02:36.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:02:36.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:02:36.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:02:36.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=20
2024-05-17T15:02:36.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:02:36.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:02:36.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:02:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:02:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:02:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:02:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:02:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:02:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:02:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:02:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:02:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 20 = ''
2024-05-17T15:02:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:02:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:02:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:37.371 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:02:37.371 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:02:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:02:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 20 = ''
2024-05-17T15:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:02:38.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:38.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:38.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:38.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:38.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:02:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:02:38.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:02:38.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:02:38.318 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:02:38.318 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:02:38.318 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:02:38.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:02:38.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 20 = ''
2024-05-17T15:02:38.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:02:38.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:02:39.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:39.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:39.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:39.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:39.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:02:39.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:02:39.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:02:39.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:02:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=20
2024-05-17T15:02:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 20 = 20
2024-05-17T15:02:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap6
2024-05-17T15:02:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6
2024-05-17T15:02:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:02:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:02:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:02:40.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap6
2024-05-17T15:02:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:02:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 12
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:00:55 2024
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:02:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:02:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:02:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:02:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap5-clean ']'
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap5-clean
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5-clean
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:02:41.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:02:41.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:41.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:41.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:02:41.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:02:41.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:43.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap6
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:43.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:43.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:02:44.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:02:44.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:02:45.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:45.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:02:45.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:45.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap6-clean
2024-05-17T15:02:45.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap6-clean
2024-05-17T15:02:45.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:45.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:45.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:45.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6-clean
2024-05-17T15:02:45.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:02:46.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:02:46.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:02:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:02:46.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:02:46.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:02:46.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:02:46.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 23001088 bytes
2024-05-17T15:02:46.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 50703360 bytes
2024-05-17T15:02:47.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 136462336 bytes
2024-05-17T15:02:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 183101440 bytes
2024-05-17T15:02:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 238827520 bytes
2024-05-17T15:02:47.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 260214784 bytes
2024-05-17T15:02:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 303358976 bytes
2024-05-17T15:02:48.026 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 327645184 bytes
2024-05-17T15:02:48.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 453783552 bytes
2024-05-17T15:02:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 503041024 bytes
2024-05-17T15:02:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 518426624 bytes
2024-05-17T15:02:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 519847936 bytes
2024-05-17T15:02:48.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 601820160 bytes
2024-05-17T15:02:49.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 633656320 bytes
2024-05-17T15:02:49.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 672117760 bytes
2024-05-17T15:02:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 733154304 bytes
2024-05-17T15:02:49.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 755580928 bytes
2024-05-17T15:02:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 813772800 bytes
2024-05-17T15:02:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 861906944 bytes
2024-05-17T15:02:50.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 895643648 bytes
2024-05-17T15:02:50.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 936330240 bytes
2024-05-17T15:02:50.398 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 967009280 bytes
2024-05-17T15:02:50.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1005149184 bytes
2024-05-17T15:02:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1005768704 bytes
2024-05-17T15:02:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1053268992 bytes
2024-05-17T15:02:51.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1113606144 bytes
2024-05-17T15:02:51.159 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1128530944 bytes
2024-05-17T15:02:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1201923072 bytes
2024-05-17T15:02:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1271271424 bytes
2024-05-17T15:02:51.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1289592832 bytes
2024-05-17T15:02:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1312403456 bytes
2024-05-17T15:02:52.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1356690432 bytes
2024-05-17T15:02:52.136 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1371524096 bytes
2024-05-17T15:02:52.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1382876160 bytes
2024-05-17T15:02:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1438646272 bytes
2024-05-17T15:02:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1449993216 bytes
2024-05-17T15:02:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1531621376 bytes
2024-05-17T15:02:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1575122944 bytes
2024-05-17T15:02:53.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1631787008 bytes
2024-05-17T15:02:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1656097792 bytes
2024-05-17T15:02:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1715458048 bytes
2024-05-17T15:02:53.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1784799232 bytes
2024-05-17T15:02:53.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1834305536 bytes
2024-05-17T15:02:54.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1849249792 bytes
2024-05-17T15:02:54.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1950078976 bytes
2024-05-17T15:02:54.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1975394304 bytes
2024-05-17T15:02:54.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1979393024 bytes
2024-05-17T15:02:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2077853696 bytes
2024-05-17T15:02:54.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2100276224 bytes
2024-05-17T15:02:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2162049024 bytes
2024-05-17T15:02:55.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2205977600 bytes
2024-05-17T15:02:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2239931392 bytes
2024-05-17T15:02:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2319817728 bytes
2024-05-17T15:02:55.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2356889600 bytes
2024-05-17T15:02:56.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8686344144558034) > weight (0.25)
2024-05-17T15:02:56.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7308843820548987) > weight (0.25)
2024-05-17T15:02:56.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.45537837548990334) > weight (0.25)
2024-05-17T15:02:56.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7630946521560382) > weight (0.25)
2024-05-17T15:02:56.000 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing
2024-05-17T15:02:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2396357632 bytes
2024-05-17T15:02:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2478075904 bytes
2024-05-17T15:02:56.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2509650944 bytes
2024-05-17T15:02:56.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2523311104 bytes
2024-05-17T15:02:56.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2559845376 bytes
2024-05-17T15:02:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2590137344 bytes
2024-05-17T15:02:57.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2660288512 bytes
2024-05-17T15:02:57.250 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2673608704 bytes
2024-05-17T15:02:57.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2741272576 bytes
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:57.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:02:57.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:02:57.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:02:57.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:02:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:02:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:02:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:02:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:02:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:02:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":17476266.67,"bytes_per_snapshot":534487040.0,"last_snapshot_bytes":534487040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958155,"remote_snapshot_timestamp":1715958155,"replay_state":"idle"}
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.3 on smithi171
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:02:41
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:02:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:02:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:02:48'
2024-05-17T15:02:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:02:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:02:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:02:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":17476266.67,"bytes_per_snapshot":534487040.0,"last_snapshot_bytes":534487040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958155,"remote_snapshot_timestamp":1715958155,"replay_state":"idle"}
2024-05-17T15:02:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:02:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:02:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:02:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:02:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.3 on smithi171
2024-05-17T15:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:02:41'
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 15:02:41'
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 15:02:41'
2024-05-17T15:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:02:58.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:02:58.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:02:58.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4449</service_id><instance_id>4451</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4516</service_id><instance_id>4519</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:17476266.67,&quot;bytes_per_snapshot&quot;:534487040.0,&quot;last_snapshot_bytes&quot;:534487040,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715958155,&quot;remote_snapshot_timestamp&quot;:1715958155,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4228</service_id><instance_id>4325</instance_id><daemon_id>mirror.3</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 15:02:41</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:02:48</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:02:58.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:02:58.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:02:58.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:02:41'
2024-05-17T15:02:58.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:02:58.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:02:58.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:02:58.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:02:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:02:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:02:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap7
2024-05-17T15:02:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap7
2024-05-17T15:02:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:02:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:02:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:02:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7
2024-05-17T15:02:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:03:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:03:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:03:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:03:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:03:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:03:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:03:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:03:29.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:03:29.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:03:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:03:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:03:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:03:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:03:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:03:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:03:29.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:03:29.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:03:29.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:03:29.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:03:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:03:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:03:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:03:29.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:03:29.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:03:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:03:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:03:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:03:29.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:03:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:03:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:03:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:03:29.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T15:03:29.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:03:29.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:03:29.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:03:29.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:03:29.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T15:03:29.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T15:03:29.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T15:03:29.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T15:03:29.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T15:03:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:03:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:03:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:03:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T15:03:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:03:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:03:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:03:29.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T15:03:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:03:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T15:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T15:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T15:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T15:03:29.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:03:29.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:03:29.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:03:29.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T15:03:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:03:29.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:03:29.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:03:29.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:03:29.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:03:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:03:29.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:03:29.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:03:29.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:29.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:03:29.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:29.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:29.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:29.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:03:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 23
2024-05-17T15:03:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:03:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:03:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:03:31.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:03:31.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:03:31.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=23
2024-05-17T15:03:31.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:03:31.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:03:31.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:03:31.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:31.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:31.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:31.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:31.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:31.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:03:31.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:03:31.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:03:31.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:03:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:03:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:03:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:03:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:03:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 23 = ''
2024-05-17T15:03:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:03:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:03:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:03:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:03:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:03:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:03:32.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:03:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:03:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:03:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:03:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 23 = ''
2024-05-17T15:03:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:03:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:03:32.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:32.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:32.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:32.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:32.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:32.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:03:32.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:03:33.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:03:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:03:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:03:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:03:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:03:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:03:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 23 = ''
2024-05-17T15:03:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:03:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:03:34.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:34.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:34.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:34.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:34.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:03:34.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:03:34.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:03:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:03:34.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=23
2024-05-17T15:03:34.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 23 = 23
2024-05-17T15:03:34.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:34.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap7
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:03:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:03:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap7
2024-05-17T15:03:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 14
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:03:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:02:41 2024
2024-05-17T15:03:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:03:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:03:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:03:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:03:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:03:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:03:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap6-clean ']'
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap6-clean
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6-clean
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:03:36.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:03:36.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:37.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:03:37.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap7
2024-05-17T15:03:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7
2024-05-17T15:03:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:03:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:03:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:03:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:03:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:03:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:40.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap7-clean
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap7-clean
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7-clean
2024-05-17T15:03:40.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:03:41.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:03:41.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:03:41.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:41.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:41.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:41.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:03:41.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:03:41.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:03:41.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:03:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 25629696 bytes
2024-05-17T15:03:41.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 35180544 bytes
2024-05-17T15:03:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 111368192 bytes
2024-05-17T15:03:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 145400832 bytes
2024-05-17T15:03:42.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 173389824 bytes
2024-05-17T15:03:42.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 183749632 bytes
2024-05-17T15:03:42.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 205172736 bytes
2024-05-17T15:03:42.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 277766144 bytes
2024-05-17T15:03:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 329914368 bytes
2024-05-17T15:03:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 382611456 bytes
2024-05-17T15:03:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 485972992 bytes
2024-05-17T15:03:43.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 552419328 bytes
2024-05-17T15:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 605264896 bytes
2024-05-17T15:03:43.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 675121152 bytes
2024-05-17T15:03:44.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 713700352 bytes
2024-05-17T15:03:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 757402624 bytes
2024-05-17T15:03:44.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 839026688 bytes
2024-05-17T15:03:44.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 871738368 bytes
2024-05-17T15:03:45.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 872354816 bytes
2024-05-17T15:03:45.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 909980672 bytes
2024-05-17T15:03:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 986441728 bytes
2024-05-17T15:03:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1014316032 bytes
2024-05-17T15:03:45.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1082741760 bytes
2024-05-17T15:03:45.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1134327808 bytes
2024-05-17T15:03:46.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1170206720 bytes
2024-05-17T15:03:46.305 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1229620224 bytes
2024-05-17T15:03:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1283646464 bytes
2024-05-17T15:03:46.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1335804928 bytes
2024-05-17T15:03:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1385777152 bytes
2024-05-17T15:03:47.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1432928256 bytes
2024-05-17T15:03:47.213 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1461246976 bytes
2024-05-17T15:03:47.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1521605632 bytes
2024-05-17T15:03:47.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1604056064 bytes
2024-05-17T15:03:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1641176064 bytes
2024-05-17T15:03:48.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1709430784 bytes
2024-05-17T15:03:48.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1716031488 bytes
2024-05-17T15:03:48.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1772775424 bytes
2024-05-17T15:03:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1797284864 bytes
2024-05-17T15:03:48.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1927874560 bytes
2024-05-17T15:03:48.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1946555392 bytes
2024-05-17T15:03:49.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1967217664 bytes
2024-05-17T15:03:49.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 2005568512 bytes
2024-05-17T15:03:49.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 2065869824 bytes
2024-05-17T15:03:49.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 2120175616 bytes
2024-05-17T15:03:49.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 2237167616 bytes
2024-05-17T15:03:50.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2263220224 bytes
2024-05-17T15:03:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2269551616 bytes
2024-05-17T15:03:50.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2345606144 bytes
2024-05-17T15:03:50.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2404632576 bytes
2024-05-17T15:03:50.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2422574080 bytes
2024-05-17T15:03:50.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2449351680 bytes
2024-05-17T15:03:51.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2465554432 bytes
2024-05-17T15:03:51.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2492862464 bytes
2024-05-17T15:03:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2501332992 bytes
2024-05-17T15:03:51.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2537211904 bytes
2024-05-17T15:03:51.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2588975104 bytes
2024-05-17T15:03:51.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2597465088 bytes
2024-05-17T15:03:51.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2656869376 bytes
2024-05-17T15:03:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2724540416 bytes
2024-05-17T15:03:52.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2776307712 bytes
2024-05-17T15:03:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2846563328 bytes
2024-05-17T15:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2898603008 bytes
2024-05-17T15:03:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2899222528 bytes
2024-05-17T15:03:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2929174528 bytes
2024-05-17T15:03:53.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2948131840 bytes
2024-05-17T15:03:53.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2978330624 bytes
2024-05-17T15:03:53.374 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2987962368 bytes
2024-05-17T15:03:53.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 3037307904 bytes
2024-05-17T15:03:53.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 3090490368 bytes
2024-05-17T15:03:53.846 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 3099687936 bytes
2024-05-17T15:03:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3162277888 bytes
2024-05-17T15:03:54.163 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3184230400 bytes
2024-05-17T15:03:54.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 3219270656 bytes
2024-05-17T15:03:54.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 3282274304 bytes
2024-05-17T15:03:54.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 3299543040 bytes
2024-05-17T15:03:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3314241536 bytes
2024-05-17T15:03:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7700 IOs, 3405224960 bytes
2024-05-17T15:03:55.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7800 IOs, 3405816832 bytes
2024-05-17T15:03:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7900 IOs, 3448855552 bytes
2024-05-17T15:03:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:03:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:03:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:03:56.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:03:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:03:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:03:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:03:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:03:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":17702843.73,"bytes_per_snapshot":532786176.0,"last_snapshot_bytes":531085312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958210,"remote_snapshot_timestamp":1715958210,"replay_state":"idle"}
2024-05-17T15:03:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.3 on smithi171
2024-05-17T15:03:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:03:41
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:03:48'
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:03:56.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:03:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":17702843.73,"bytes_per_snapshot":532786176.0,"last_snapshot_bytes":531085312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958210,"remote_snapshot_timestamp":1715958210,"replay_state":"idle"}
2024-05-17T15:03:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:03:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:03:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:03:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:03:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:03:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.3 on smithi171
2024-05-17T15:03:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:03:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:03:56.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:03:41'
2024-05-17T15:03:56.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 15:03:41'
2024-05-17T15:03:56.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 15:03:41'
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:03:56.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:03:56.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4449</service_id><instance_id>4451</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4516</service_id><instance_id>4519</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:17702843.73,&quot;bytes_per_snapshot&quot;:532786176.0,&quot;last_snapshot_bytes&quot;:531085312,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715958210,&quot;remote_snapshot_timestamp&quot;:1715958210,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4228</service_id><instance_id>4325</instance_id><daemon_id>mirror.3</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 15:03:41</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:03:48</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:03:56.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:03:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:03:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:03:41'
2024-05-17T15:03:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:03:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:03:56.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:03:56.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:03:56.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap8
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap8
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:03:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8
2024-05-17T15:03:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:04:00.009 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8272018504642837) > weight (0.25)
2024-05-17T15:04:00.009 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.49482230507701275) > weight (0.25)
2024-05-17T15:04:00.010 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3638958937817268) > weight (0.25)
2024-05-17T15:04:00.010 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7865866838885555) > weight (0.25)
2024-05-17T15:04:00.010 INFO:tasks.rbd_mirror_thrash:waiting for 118 secs before thrashing
2024-05-17T15:04:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:04:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:04:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:04:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:04:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:04:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:04:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:04:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:04:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:04:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:04:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:04:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:04:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:04:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:04:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:04:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:04:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:04:26.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:04:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:04:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:04:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:04:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:04:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:04:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:04:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:04:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:04:26.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:04:26.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:04:26.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:04:26.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:04:26.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:04:26.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:04:26.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:04:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:04:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:04:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:04:26.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:04:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:04:26.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:04:26.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:04:26.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:04:26.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:04:26.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:04:26.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:04:26.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T15:04:26.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:04:26.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:04:26.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:04:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:04:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T15:04:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T15:04:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:04:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T15:04:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T15:04:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:04:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T15:04:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:04:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T15:04:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:04:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T15:04:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:04:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:04:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T15:04:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T15:04:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T15:04:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T15:04:26.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:04:26.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:04:26.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T15:04:26.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T15:04:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:04:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T15:04:26.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:04:26.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:04:26.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T15:04:26.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T15:04:26.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T15:04:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:04:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:04:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:04:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T15:04:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:04:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:04:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:04:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:04:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:04:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:04:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:04:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:04:27.097 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:04:27.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:04:27.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:04:27.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:04:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:04:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:04:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 26
2024-05-17T15:04:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:04:28.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:28.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:28.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:28.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:04:28.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:04:28.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:04:28.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:04:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=26
2024-05-17T15:04:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:04:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:04:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:04:28.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:04:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:04:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:04:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:04:28.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:04:28.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:04:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:04:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:04:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 26 = ''
2024-05-17T15:04:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:04:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:04:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:29.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:04:29.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:04:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:04:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 26 = ''
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:04:29.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:04:30.209 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:30.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:30.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:30.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:30.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:30.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:04:30.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:04:30.271 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:04:30.271 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 26 = ''
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:04:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:04:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:04:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:04:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:04:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:04:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=26
2024-05-17T15:04:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 26 = 26
2024-05-17T15:04:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap8
2024-05-17T15:04:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:31.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:31.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:31.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8
2024-05-17T15:04:31.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:04:31.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:04:31.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:04:32.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap8
2024-05-17T15:04:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:04:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:04:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:04:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 16
2024-05-17T15:04:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:04:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:02:41 2024
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:04:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:04:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap7-clean ']'
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap7-clean
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7-clean
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:33.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:04:33.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:04:33.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:34.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:04:34.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:35.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap8
2024-05-17T15:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:04:35.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:04:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:37.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:04:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:37.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:04:37.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:37.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap8-clean
2024-05-17T15:04:37.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap8-clean
2024-05-17T15:04:37.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:37.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:37.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:37.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8-clean
2024-05-17T15:04:37.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:04:38.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:04:38.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=10
2024-05-17T15:04:38.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:04:38.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:04:38.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 29654016 bytes
2024-05-17T15:04:38.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 85380096 bytes
2024-05-17T15:04:39.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 139427840 bytes
2024-05-17T15:04:39.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 145343488 bytes
2024-05-17T15:04:39.340 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 178445312 bytes
2024-05-17T15:04:39.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 274387968 bytes
2024-05-17T15:04:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 324873216 bytes
2024-05-17T15:04:39.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 330881024 bytes
2024-05-17T15:04:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 347348992 bytes
2024-05-17T15:04:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 391739392 bytes
2024-05-17T15:04:40.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 407214080 bytes
2024-05-17T15:04:40.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 457529344 bytes
2024-05-17T15:04:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 523730944 bytes
2024-05-17T15:04:40.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 542690304 bytes
2024-05-17T15:04:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 614426624 bytes
2024-05-17T15:04:41.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 643872768 bytes
2024-05-17T15:04:41.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 669363200 bytes
2024-05-17T15:04:41.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 763350016 bytes
2024-05-17T15:04:41.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 811763712 bytes
2024-05-17T15:04:41.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 851646464 bytes
2024-05-17T15:04:42.266 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 947688448 bytes
2024-05-17T15:04:42.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1027349504 bytes
2024-05-17T15:04:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1055686656 bytes
2024-05-17T15:04:43.002 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7111108939493349) > weight (0.25)
2024-05-17T15:04:43.002 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7617156990636359) > weight (0.25)
2024-05-17T15:04:43.002 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T15:04:43.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T15:04:43.003 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons
2024-05-17T15:04:43.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T15:04:43.005+0000 7fa290ae5640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 30397) UID: 1000
2024-05-17T15:04:43.048 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1075047424 bytes
2024-05-17T15:04:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1118989312 bytes
2024-05-17T15:04:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1157102592 bytes
2024-05-17T15:04:43.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1157739520 bytes
2024-05-17T15:04:43.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1185181696 bytes
2024-05-17T15:04:43.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1231141888 bytes
2024-05-17T15:04:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1246172160 bytes
2024-05-17T15:04:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1265008640 bytes
2024-05-17T15:04:44.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1307581440 bytes
2024-05-17T15:04:44.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1387904000 bytes
2024-05-17T15:04:44.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1390949376 bytes
2024-05-17T15:04:44.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1404796928 bytes
2024-05-17T15:04:44.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1473212416 bytes
2024-05-17T15:04:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1541980160 bytes
2024-05-17T15:04:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1568925696 bytes
2024-05-17T15:04:45.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1597607936 bytes
2024-05-17T15:04:45.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1619312640 bytes
2024-05-17T15:04:45.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1643418624 bytes
2024-05-17T15:04:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1681772544 bytes
2024-05-17T15:04:46.003 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T15:04:46.003 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:04:46.004 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T15:04:46.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T15:04:46.004 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T15:04:46.007 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T15:04:46.007 INFO:tasks.rbd_mirror_thrash:waiting for 84 secs before thrashing
2024-05-17T15:04:46.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1743833088 bytes
2024-05-17T15:04:46.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1794679808 bytes
2024-05-17T15:04:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1851822080 bytes
2024-05-17T15:04:46.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1882391552 bytes
2024-05-17T15:04:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1901425664 bytes
2024-05-17T15:04:46.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1936789504 bytes
2024-05-17T15:04:47.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1993734144 bytes
2024-05-17T15:04:47.212 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2031976448 bytes
2024-05-17T15:04:47.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2073222144 bytes
2024-05-17T15:04:47.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2160550912 bytes
2024-05-17T15:04:47.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2257660928 bytes
2024-05-17T15:04:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2306305024 bytes
2024-05-17T15:04:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2334746624 bytes
2024-05-17T15:04:48.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2380742656 bytes
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:04:48.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:04:48.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:04:48.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:04:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:04:49.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:04:49.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":17871940.27,"bytes_per_snapshot":533621760.0,"last_snapshot_bytes":536158208,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958267,"remote_snapshot_timestamp":1715958267,"replay_state":"idle"}
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.3 on smithi171
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:04:41
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:04:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:04:47'
2024-05-17T15:04:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:04:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:04:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:04:49.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":17871940.27,"bytes_per_snapshot":533621760.0,"last_snapshot_bytes":536158208,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958267,"remote_snapshot_timestamp":1715958267,"replay_state":"idle"}
2024-05-17T15:04:49.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:04:49.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:04:49.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:04:49.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:04:49.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:04:49.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.3 on smithi171
2024-05-17T15:04:49.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:04:49.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:04:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:04:41'
2024-05-17T15:04:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 15:04:41'
2024-05-17T15:04:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:04:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:04:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 15:04:41'
2024-05-17T15:04:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:04:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:04:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:04:49.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4449</service_id><instance_id>4451</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4516</service_id><instance_id>4519</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:17871940.27,&quot;bytes_per_snapshot&quot;:533621760.0,&quot;last_snapshot_bytes&quot;:536158208,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715958267,&quot;remote_snapshot_timestamp&quot;:1715958267,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4228</service_id><instance_id>4325</instance_id><daemon_id>mirror.3</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 15:04:41</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:04:47</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:04:49.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:04:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:04:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:04:41'
2024-05-17T15:04:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:04:49.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:04:49.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:04:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:04:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:04:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:04:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap9
2024-05-17T15:04:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap9
2024-05-17T15:04:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:04:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:04:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:04:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9
2024-05-17T15:04:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:05:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:05:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:05:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:05:20.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:05:20.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:20.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:20.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:20.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:05:20.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:05:20.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:05:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:05:20.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:05:20.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:05:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:05:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:05:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:05:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:05:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:05:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:05:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:05:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:05:20.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:05:20.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:05:20.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:05:20.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:05:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:05:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:05:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:05:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:05:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:05:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:05:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:05:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:05:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:05:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:05:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T15:05:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:05:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:05:20.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:05:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:05:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:05:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:05:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:05:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T15:05:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T15:05:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T15:05:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:05:20.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:05:20.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:05:20.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T15:05:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:05:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:05:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:05:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:05:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T15:05:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T15:05:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T15:05:20.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T15:05:20.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T15:05:20.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T15:05:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:05:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:05:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:05:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T15:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:05:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:05:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:05:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:05:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:05:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:05:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:21.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:05:21.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:21.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:21.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:21.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:05:22.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 29
2024-05-17T15:05:22.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:05:22.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:05:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:05:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:05:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:05:22.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=29
2024-05-17T15:05:22.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:05:22.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:05:22.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:05:22.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:05:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:05:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:05:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:05:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:05:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:05:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:05:22.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:05:22.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 29 = ''
2024-05-17T15:05:22.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:05:22.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:05:23.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:23.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:23.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:23.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:23.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:23.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:05:23.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:05:23.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:05:23.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:05:23.399 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:05:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:05:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:05:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:05:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 29 = ''
2024-05-17T15:05:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:05:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:05:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:05:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:05:24.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:05:24.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:05:24.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:05:24.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:05:24.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:05:24.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:05:24.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 29 = ''
2024-05-17T15:05:24.281 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:05:24.281 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:05:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:05:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:05:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:05:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:05:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=29
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 29 = 29
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap9
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:05:25.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:05:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap9
2024-05-17T15:05:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:05:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:05:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:05:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 18
2024-05-17T15:05:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:05:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:04:33 2024
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:05:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap8-clean ']'
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap8-clean
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8-clean
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:05:27.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:05:27.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:28.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:05:28.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:29.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:29.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap9
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:05:30.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:05:30.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:31.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:05:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:31.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap9-clean
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap9-clean
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9-clean
2024-05-17T15:05:31.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:05:32.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:05:32.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror test
2024-05-17T15:05:32.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:32.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:32.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:32.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:05:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:05:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:05:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20
2024-05-17T15:05:33.374 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 18626560 bytes
2024-05-17T15:05:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 71597056 bytes
2024-05-17T15:05:33.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 125291520 bytes
2024-05-17T15:05:34.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 213105664 bytes
2024-05-17T15:05:34.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 215604224 bytes
2024-05-17T15:05:34.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 266617856 bytes
2024-05-17T15:05:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 302232576 bytes
2024-05-17T15:05:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 369463296 bytes
2024-05-17T15:05:34.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 390926336 bytes
2024-05-17T15:05:35.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 443222016 bytes
2024-05-17T15:05:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 528179200 bytes
2024-05-17T15:05:35.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 553932800 bytes
2024-05-17T15:05:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 619866112 bytes
2024-05-17T15:05:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 668742656 bytes
2024-05-17T15:05:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 688268288 bytes
2024-05-17T15:05:36.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 708106240 bytes
2024-05-17T15:05:36.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 757406720 bytes
2024-05-17T15:05:36.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 802591744 bytes
2024-05-17T15:05:36.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 831110144 bytes
2024-05-17T15:05:36.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 869840896 bytes
2024-05-17T15:05:36.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 909493248 bytes
2024-05-17T15:05:37.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 972798976 bytes
2024-05-17T15:05:37.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1041187840 bytes
2024-05-17T15:05:37.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1041811456 bytes
2024-05-17T15:05:37.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1073000448 bytes
2024-05-17T15:05:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1089248256 bytes
2024-05-17T15:05:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1109555200 bytes
2024-05-17T15:05:38.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1147474944 bytes
2024-05-17T15:05:38.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1166045184 bytes
2024-05-17T15:05:38.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1245381632 bytes
2024-05-17T15:05:38.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1282434048 bytes
2024-05-17T15:05:38.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1298774016 bytes
2024-05-17T15:05:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1342692352 bytes
2024-05-17T15:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1360346112 bytes
2024-05-17T15:05:39.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1441283072 bytes
2024-05-17T15:05:39.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1516608512 bytes
2024-05-17T15:05:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1573218304 bytes
2024-05-17T15:05:39.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1593527296 bytes
2024-05-17T15:05:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1650754560 bytes
2024-05-17T15:05:40.230 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1705737216 bytes
2024-05-17T15:05:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1737225216 bytes
2024-05-17T15:05:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1738870784 bytes
2024-05-17T15:05:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1759017984 bytes
2024-05-17T15:05:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1786905600 bytes
2024-05-17T15:05:40.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1839650816 bytes
2024-05-17T15:05:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1859883008 bytes
2024-05-17T15:05:41.234 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1920675840 bytes
2024-05-17T15:05:41.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1937680384 bytes
2024-05-17T15:05:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1985731584 bytes
2024-05-17T15:05:41.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2068175872 bytes
2024-05-17T15:05:41.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2068777984 bytes
2024-05-17T15:05:42.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2213865472 bytes
2024-05-17T15:05:42.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2239096832 bytes
2024-05-17T15:05:42.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2275705856 bytes
2024-05-17T15:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2308078592 bytes
2024-05-17T15:05:42.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2375987200 bytes
2024-05-17T15:05:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2438460416 bytes
2024-05-17T15:05:43.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2466203648 bytes
2024-05-17T15:05:43.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2526972928 bytes
2024-05-17T15:05:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2578660352 bytes
2024-05-17T15:05:43.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2621748224 bytes
2024-05-17T15:05:43.983 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2646409216 bytes
2024-05-17T15:05:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2719600640 bytes
2024-05-17T15:05:44.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2766860288 bytes
2024-05-17T15:05:44.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2771881984 bytes
2024-05-17T15:05:44.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2805336064 bytes
2024-05-17T15:05:44.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2850869248 bytes
2024-05-17T15:05:44.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2882069504 bytes
2024-05-17T15:05:45.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2935279616 bytes
2024-05-17T15:05:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2975632384 bytes
2024-05-17T15:05:45.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3011411968 bytes
2024-05-17T15:05:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:05:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' ''
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local description_pattern=
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local service_pattern=
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:05:46.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:05:46.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror image status mirror/test
2024-05-17T15:05:46.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='test:
2024-05-17T15:05:46.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  global_id:   ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  state:       up+replaying
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  description: replaying, {"bytes_per_second":17833608.53,"bytes_per_snapshot":535583232.0,"last_snapshot_bytes":535008256,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958321,"remote_snapshot_timestamp":1715958321,"replay_state":"idle"}
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  service:     mirror.3 on smithi171
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  last_update: 2024-05-17 15:05:41
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:  peer_sites:
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    name: cluster2
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    state: up+stopped
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    description: local image is primary
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:    last_update: 2024-05-17 15:05:41'
2024-05-17T15:05:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  state: .*up+replaying'
2024-05-17T15:05:46.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  state:       up+replaying
2024-05-17T15:05:46.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^  description: .*'
2024-05-17T15:05:46.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  description: replaying, {"bytes_per_second":17833608.53,"bytes_per_snapshot":535583232.0,"last_snapshot_bytes":535008256,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1715958321,"remote_snapshot_timestamp":1715958321,"replay_state":"idle"}
2024-05-17T15:05:46.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:05:46.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:05:46.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep '^up+'
2024-05-17T15:05:46.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:05:46.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep 'service: *mirror..* on '
2024-05-17T15:05:46.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:  service:     mirror.3 on smithi171
2024-05-17T15:05:46.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update
2024-05-17T15:05:46.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -nEe 's/^  last_update: *(.*) *$/\1/p'
2024-05-17T15:05:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:05:41'
2024-05-17T15:05:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-05-17 15:05:41'
2024-05-17T15:05:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:05:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state_pattern=up+replaying
2024-05-17T15:05:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local 'prev_last_update=2024-05-17 15:05:41'
2024-05-17T15:05:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status
2024-05-17T15:05:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T15:05:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml
2024-05-17T15:05:46.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status='<status><summary><health>OK</health><daemon_health>OK</daemon_health><image_health>OK</image_health><states><replaying>1</replaying></states></summary><daemons><daemon><service_id>4228</service_id><instance_id>4325</instance_id><client_id>mirror.3</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>true</leader><health>OK</health></daemon><daemon><service_id>4385</service_id><instance_id>4388</instance_id><client_id>mirror.1</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4449</service_id><instance_id>4451</instance_id><client_id>mirror.2</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon><daemon><service_id>4516</service_id><instance_id>4519</instance_id><client_id>mirror.0</client_id><hostname>smithi171</hostname><ceph_version>19.0.0-3735-gee59d86a</ceph_version><leader>false</leader><health>OK</health></daemon></daemons><images><image><name>test</name><global_id>ce196974-fb71-4415-bf93-e590315f06b5</global_id><state>up+replaying</state><description>replaying, {&quot;bytes_per_second&quot;:17833608.53,&quot;bytes_per_snapshot&quot;:535583232.0,&quot;last_snapshot_bytes&quot;:535008256,&quot;last_snapshot_sync_seconds&quot;:1,&quot;local_snapshot_timestamp&quot;:1715958321,&quot;remote_snapshot_timestamp&quot;:1715958321,&quot;replay_state&quot;:&quot;idle&quot;}</description><daemon_service><service_id>4228</service_id><instance_id>4325</instance_id><daemon_id>mirror.3</daemon_id><hostname>smithi171</hostname></daemon_service><last_update>2024-05-17 15:05:41</last_update><peer_sites><peer_site><site_name>cluster2</site_name><mirror_uuids>c25caed9-298e-45f3-a659-2da8c449ce51</mirror_uuids><state>up+stopped</state><description>local image is primary</description><last_update>2024-05-17 15:05:41</last_update></peer_site></peer_sites></image></images></status>'
2024-05-17T15:05:46.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_update state
2024-05-17T15:05:46.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update'
2024-05-17T15:05:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ last_update='2024-05-17 15:05:41'
2024-05-17T15:05:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state'
2024-05-17T15:05:46.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=up+replaying
2024-05-17T15:05:46.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo up+replaying
2024-05-17T15:05:46.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep up+replaying
2024-05-17T15:05:46.753 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:up+replaying
2024-05-17T15:05:46.753 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap10
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap10
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10
2024-05-17T15:05:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:05:58.011 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5739770224867328) > weight (0.25)
2024-05-17T15:05:58.012 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1
2024-05-17T15:05:58.012 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15
2024-05-17T15:05:58.012 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons
2024-05-17T15:05:58.013 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi171.stderr:2024-05-17T15:05:58.015+0000 7f6806db8640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1  (PID: 31599) UID: 1000
2024-05-17T15:06:07.013 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1
2024-05-17T15:06:07.013 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:06:07.013 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1
2024-05-17T15:06:07.014 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon
2024-05-17T15:06:07.014 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1
2024-05-17T15:06:07.021 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started
2024-05-17T15:06:07.021 INFO:tasks.rbd_mirror_thrash:waiting for 65 secs before thrashing
2024-05-17T15:06:10.008 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.25107338356530673) > weight (0.25)
2024-05-17T15:06:10.008 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9772408730609115) > weight (0.25)
2024-05-17T15:06:10.008 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.40621499878369816) > weight (0.25)
2024-05-17T15:06:10.008 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3
2024-05-17T15:06:10.008 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15
2024-05-17T15:06:10.009 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons
2024-05-17T15:06:10.010 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi171.stderr:2024-05-17T15:06:10.011+0000 7f3253a38640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3  (PID: 29358) UID: 1000
2024-05-17T15:06:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:06:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:06:15.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:06:15.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:06:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:06:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:06:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:06:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:06:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:06:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:06:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:06:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:06:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:06:15.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:06:15.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:06:15.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:06:15.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:06:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:06:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:06:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:06:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:06:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:06:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:06:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:06:15.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:06:15.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:06:15.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:06:15.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T15:06:15.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:06:15.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:06:15.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:06:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T15:06:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:15.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T15:06:15.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T15:06:15.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T15:06:15.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:06:15.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:06:15.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:06:15.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T15:06:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:06:15.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:06:15.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:06:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T15:06:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T15:06:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T15:06:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:15.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T15:06:15.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T15:06:15.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T15:06:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:06:15.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:06:15.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:06:15.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T15:06:15.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:06:15.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:06:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:06:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:06:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:06:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:15.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:15.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:06:15.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:06:15.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:06:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:06:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:06:15.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:06:15.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:06:15.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:06:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:06:16.009 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3
2024-05-17T15:06:16.010 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:06:16.010 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3
2024-05-17T15:06:16.010 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon
2024-05-17T15:06:16.010 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3
2024-05-17T15:06:16.013 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started
2024-05-17T15:06:16.013 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing
2024-05-17T15:06:17.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 32
2024-05-17T15:06:17.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:06:17.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:17.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:17.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:17.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:06:17.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:17.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:06:17.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:06:17.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=32
2024-05-17T15:06:17.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:06:17.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:06:17.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:06:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:17.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:06:17.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:17.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:06:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:06:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:06:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:06:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:06:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:06:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 32 = ''
2024-05-17T15:06:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:06:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:06:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:18.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:06:18.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:06:18.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:06:18.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:06:18.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:06:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:06:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 32 = ''
2024-05-17T15:06:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:06:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:06:18.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:18.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:18.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:18.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:18.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:18.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:18.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:06:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:06:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 32 = ''
2024-05-17T15:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:06:20.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:20.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:20.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:20.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:20.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:20.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:06:20.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:06:20.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=32
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 32 = 32
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap10
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:06:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:06:21.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap10
2024-05-17T15:06:21.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:06:21.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:06:21.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 20
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:04:33 2024
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:06:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:06:21.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snap9-clean ']'
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap9-clean
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9-clean
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:21.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:22.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:22.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:23.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:23.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:24.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:24.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap10
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:24.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:24.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:26.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:26.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:26.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ clean_snap_name=snap10-clean
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror test snap10-clean
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10-clean
2024-05-17T15:06:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20
2024-05-17T15:06:27.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror test
2024-05-17T15:06:27.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:27.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:27.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:27.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror test started
2024-05-17T15:06:27.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:27.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:27.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:27.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:06:27.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:06:27.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:06:27.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:06:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror test started
2024-05-17T15:06:28.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:28.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:28.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:28.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:06:28.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:06:28.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:06:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:06:28.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:06:28.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:06:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test
2024-05-17T15:06:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:06:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:06:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:06:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:06:28.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:06:28.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:06:28.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:06:28.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test
2024-05-17T15:06:28.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:06:28.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:06:28.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:28.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:06:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:06:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:06:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:06:28.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:06:28.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:06:28.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:06:28.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test
2024-05-17T15:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:2
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:2 cluster instance
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:2
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=2
2024-05-17T15:06:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 2 = cluster1:2 ']'
2024-05-17T15:06:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=2
2024-05-17T15:06:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:2 cluster1
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:2
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:2
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=2
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 2 = cluster1:2 ']'
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=2
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=2
2024-05-17T15:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket'
2024-05-17T15:06:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:06:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:06:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok
2024-05-17T15:06:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test
2024-05-17T15:06:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/test']
2024-05-17T15:06:28.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:06:28.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:06:28.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:06:28.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test
2024-05-17T15:06:28.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:3
2024-05-17T15:06:28.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:3 cluster instance
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:3
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=3
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 3 = cluster1:3 ']'
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=3
2024-05-17T15:06:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:3 cluster1
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:3
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:3
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=3
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 3 = cluster1:3 ']'
2024-05-17T15:06:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:06:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:06:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=3
2024-05-17T15:06:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=3
2024-05-17T15:06:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket'
2024-05-17T15:06:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:06:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:06:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok
2024-05-17T15:06:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test
2024-05-17T15:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:06:28.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:06:28.979 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:06:28.979 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:06:28.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:06:28.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-test.status
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-test.status
2024-05-17T15:06:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:06:28.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:06:28.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-test.status
2024-05-17T15:06:28.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-test.status
2024-05-17T15:06:28.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:06:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror test
2024-05-17T15:06:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test
2024-05-17T15:06:30.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 34
2024-05-17T15:06:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:06:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:06:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.status
2024-05-17T15:06:30.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml
2024-05-17T15:06:30.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:30.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:06:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=34
2024-05-17T15:06:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:06:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:06:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:06:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.status
2024-05-17T15:06:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml
2024-05-17T15:06:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:06:30.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=34
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 34 = 34
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10-clean
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:06:30.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'test':
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 512 MiB in 128 objects
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 21
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 10ceb764e0d2
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.10ceb764e0d2
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:06:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:06:22 2024
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 14:57:23 2024
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce196974-fb71-4415-bf93-e590315f06b5
2024-05-17T15:06:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:06:31.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:31.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 10
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap1
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap1
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:32.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:32.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:33.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:33.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:34.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap1-clean
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap1-clean
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap1-clean
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:34.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:35.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:35.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:35.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap2
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap2
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:36.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2
2024-05-17T15:06:36.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:36.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:36.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:36.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:36.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:38.290 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:38.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:38.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap2-clean
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap2-clean
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap2-clean
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:38.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:39.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:40.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:40.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap3
2024-05-17T15:06:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap3
2024-05-17T15:06:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3
2024-05-17T15:06:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap3-clean
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap3-clean
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap3-clean
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:43.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:43.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:43.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:45.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:45.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:45.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap4
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap4
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:46.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:47.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap4-clean
2024-05-17T15:06:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap4-clean
2024-05-17T15:06:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap4-clean
2024-05-17T15:06:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:48.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:49.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:49.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:50.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap5
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap5
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:50.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap5-clean
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap5-clean
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap5-clean
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:53.231 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:54.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:54.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap6
2024-05-17T15:06:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap6
2024-05-17T15:06:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:56.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:56.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:57.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:57.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap6-clean
2024-05-17T15:06:57.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap6-clean
2024-05-17T15:06:57.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap6-clean
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:57.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:58.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:59.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:06:59.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:59.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap7
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap7
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:00.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:01.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:01.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap7-clean
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap7-clean
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap7-clean
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:04.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:04.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap8
2024-05-17T15:07:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap8
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:05.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:06.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:06.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap8-clean
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap8-clean
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap8-clean
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:06.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:07.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:07.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:08.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:08.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:09.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap9
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap9
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:09.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:09.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:11.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:11.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap9-clean
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap9-clean
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap9-clean
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:11.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:12.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4988114719332043) > weight (0.25)
2024-05-17T15:07:12.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.42983708572831325) > weight (0.25)
2024-05-17T15:07:12.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.847747767395803) > weight (0.25)
2024-05-17T15:07:12.022 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3
2024-05-17T15:07:12.022 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15
2024-05-17T15:07:12.023 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons
2024-05-17T15:07:12.023 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi171.stderr:2024-05-17T15:07:12.022+0000 7f5dfab0b640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3  (PID: 29314) UID: 1000
2024-05-17T15:07:12.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:13.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:13.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:13.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap10
2024-05-17T15:07:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap10
2024-05-17T15:07:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10
2024-05-17T15:07:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:14.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:14.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:15.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:15.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:15.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:16.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap10-clean
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror test snap10-clean
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap10-clean
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:16.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:16.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export
2024-05-17T15:07:16.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:17.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 1% complete...
Exporting image: 2% complete...
Exporting image: 3% complete...
Exporting image: 4% complete...
Exporting image: 5% complete...
Exporting image: 6% complete...
Exporting image: 7% complete...
Exporting image: 8% complete...
Exporting image: 9% complete...
Exporting image: 10% complete...
Exporting image: 11% complete...
Exporting image: 12% complete...
Exporting image: 13% complete...
Exporting image: 14% complete...
Exporting image: 15% complete...
Exporting image: 16% complete...
Exporting image: 17% complete...
Exporting image: 18% complete...
Exporting image: 19% complete...
Exporting image: 20% complete...
Exporting image: 21% complete...
Exporting image: 22% complete...
Exporting image: 23% complete...
Exporting image: 24% complete...
Exporting image: 25% complete...
Exporting image: 26% complete...
Exporting image: 27% complete...
Exporting image: 28% complete...
Exporting image: 29% complete...
Exporting image: 30% complete...
Exporting image: 31% complete...
Exporting image: 32% complete...
Exporting image: 33% complete...
Exporting image: 34% complete...
Exporting image: 35% complete...
Exporting image: 36% complete...
Exporting image: 37% complete...
Exporting image: 38% complete...
Exporting image: 39% complete...
Exporting image: 40% complete...
Exporting image: 41% complete...
Exporting image: 42% complete...
Exporting image: 43% complete...
Exporting image: 44% complete...
Exporting image: 45% complete...
Exporting image: 46% complete...
Exporting image: 47% complete...
Exporting image: 48% complete...
Exporting image: 49% complete...
Exporting image: 50% complete...
Exporting image: 51% complete...
Exporting image: 52% complete...
Exporting image: 53% complete...
Exporting image: 54% complete...
Exporting image: 55% complete...
Exporting image: 56% complete...
Exporting image: 57% complete...
Exporting image: 58% complete...
Exporting image: 59% complete...
Exporting image: 60% complete...
Exporting image: 61% complete...
Exporting image: 62% complete...
Exporting image: 63% complete...
Exporting image: 64% complete...
Exporting image: 65% complete...
Exporting image: 66% complete...
Exporting image: 67% complete...
Exporting image: 68% complete...
Exporting image: 69% complete...
Exporting image: 70% complete...
Exporting image: 71% complete...
Exporting image: 72% complete...
Exporting image: 73% complete...
Exporting image: 74% complete...
Exporting image: 75% complete...
Exporting image: 76% complete...
Exporting image: 77% complete...
Exporting image: 78% complete...
Exporting image: 79% complete...
Exporting image: 80% complete...
Exporting image: 81% complete...
Exporting image: 82% complete...
Exporting image: 83% complete...
Exporting image: 84% complete...
Exporting image: 85% complete...
Exporting image: 86% complete...
Exporting image: 87% complete...
Exporting image: 88% complete...
Exporting image: 89% complete...
Exporting image: 90% complete...
Exporting image: 91% complete...
Exporting image: 92% complete...
Exporting image: 93% complete...
Exporting image: 94% complete...
Exporting image: 95% complete...
Exporting image: 96% complete...
Exporting image: 97% complete...
Exporting image: 98% complete...
Exporting image: 99% complete...
Exporting image: 100% complete...done.
2024-05-17T15:07:17.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:18.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-test.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-test.export
2024-05-17T15:07:18.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 10
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap1
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap1
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap1
2024-05-17T15:07:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1
2024-05-17T15:07:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap1-clean
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap1-clean
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap1-clean
2024-05-17T15:07:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean
2024-05-17T15:07:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:19.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap2
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap2
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap2
2024-05-17T15:07:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2
2024-05-17T15:07:20.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap2-clean
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap2-clean
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap2-clean
2024-05-17T15:07:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean
2024-05-17T15:07:21.023 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3
2024-05-17T15:07:21.024 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:07:21.024 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3
2024-05-17T15:07:21.024 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon
2024-05-17T15:07:21.024 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3
2024-05-17T15:07:21.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started
2024-05-17T15:07:21.027 INFO:tasks.rbd_mirror_thrash:waiting for 118 secs before thrashing
2024-05-17T15:07:21.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap3
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap3
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap3
2024-05-17T15:07:21.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3
2024-05-17T15:07:22.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap3-clean
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap3-clean
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap3-clean
2024-05-17T15:07:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean
2024-05-17T15:07:23.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:23.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap4
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap4
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap4
2024-05-17T15:07:23.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4
2024-05-17T15:07:24.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:24.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap4-clean
2024-05-17T15:07:24.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap4-clean
2024-05-17T15:07:24.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:24.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:24.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:24.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap4-clean
2024-05-17T15:07:24.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean
2024-05-17T15:07:25.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap5
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap5
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap5
2024-05-17T15:07:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5
2024-05-17T15:07:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap5-clean
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap5-clean
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap5-clean
2024-05-17T15:07:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean
2024-05-17T15:07:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap6
2024-05-17T15:07:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap6
2024-05-17T15:07:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap6
2024-05-17T15:07:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6
2024-05-17T15:07:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap6-clean
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap6-clean
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap6-clean
2024-05-17T15:07:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean
2024-05-17T15:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:29.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:29.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap7
2024-05-17T15:07:29.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap7
2024-05-17T15:07:29.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:29.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:29.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:29.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap7
2024-05-17T15:07:29.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7
2024-05-17T15:07:30.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap7-clean
2024-05-17T15:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap7-clean
2024-05-17T15:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap7-clean
2024-05-17T15:07:30.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean
2024-05-17T15:07:31.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap8
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap8
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap8
2024-05-17T15:07:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8
2024-05-17T15:07:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap8-clean
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap8-clean
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap8-clean
2024-05-17T15:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean
2024-05-17T15:07:33.625 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:33.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:33.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap9
2024-05-17T15:07:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap9
2024-05-17T15:07:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap9
2024-05-17T15:07:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9
2024-05-17T15:07:34.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap9-clean
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap9-clean
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap9-clean
2024-05-17T15:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean
2024-05-17T15:07:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 10`
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap10
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap10
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap10
2024-05-17T15:07:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10
2024-05-17T15:07:36.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap10-clean
2024-05-17T15:07:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_snapshot cluster2 mirror test snap10-clean
2024-05-17T15:07:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:36.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap=snap10-clean
2024-05-17T15:07:36.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean
2024-05-17T15:07:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing snap: 100% complete...done.
2024-05-17T15:07:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_image_retry cluster2 mirror test
2024-05-17T15:07:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0 1 2 4 8 16 32
2024-05-17T15:07:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0
2024-05-17T15:07:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ remove_image cluster2 mirror test
2024-05-17T15:07:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 snap purge mirror/test
2024-05-17T15:07:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 rm mirror/test
2024-05-17T15:07:37.686 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T15:07:37.686+0000 7f9cb0062640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558ce6c3fa00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted
2024-05-17T15:07:37.686 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T15:07:37.686+0000 7f9cb686f640 -1 rbd::mirror::ImageReplayer: 0x558ce6ec8280 [3/ce196974-fb71-4415-bf93-e590315f06b5] operator(): start failed: (85) Interrupted system call should be restarted
2024-05-17T15:07:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Removing image: 1% complete...
Removing image: 2% complete...
Removing image: 3% complete...
Removing image: 4% complete...
Removing image: 5% complete...
Removing image: 6% complete...
Removing image: 7% complete...
Removing image: 8% complete...
Removing image: 9% complete...
Removing image: 10% complete...
Removing image: 11% complete...
Removing image: 12% complete...
Removing image: 13% complete...
Removing image: 14% complete...
Removing image: 15% complete...
Removing image: 16% complete...
Removing image: 17% complete...
Removing image: 18% complete...
Removing image: 19% complete...
Removing image: 20% complete...
Removing image: 21% complete...
Removing image: 22% complete...
Removing image: 23% complete...
Removing image: 24% complete...
Removing image: 25% complete...
Removing image: 26% complete...
Removing image: 27% complete...
Removing image: 28% complete...
Removing image: 29% complete...
Removing image: 30% complete...
Removing image: 31% complete...
Removing image: 32% complete...
Removing image: 33% complete...
Removing image: 34% complete...
Removing image: 35% complete...
Removing image: 36% complete...
Removing image: 37% complete...
Removing image: 38% complete...
Removing image: 39% complete...
Removing image: 40% complete...
Removing image: 41% complete...
Removing image: 42% complete...
Removing image: 43% complete...
Removing image: 44% complete...
Removing image: 45% complete...
Removing image: 46% complete...
Removing image: 47% complete...
Removing image: 48% complete...
Removing image: 49% complete...
Removing image: 50% complete...
Removing image: 51% complete...
Removing image: 52% complete...
Removing image: 53% complete...
Removing image: 54% complete...
Removing image: 55% complete...
Removing image: 56% complete...
Removing image: 57% complete...
Removing image: 58% complete...
Removing image: 59% complete...
Removing image: 60% complete...
Removing image: 61% complete...
Removing image: 62% complete...
Removing image: 63% complete...
Removing image: 64% complete...
Removing image: 65% complete...
Removing image: 66% complete...
Removing image: 67% complete...
Removing image: 68% complete...
Removing image: 69% complete...
Removing image: 70% complete...
Removing image: 71% complete...
Removing image: 72% complete...
Removing image: 73% complete...
Removing image: 74% complete...
Removing image: 75% complete...
Removing image: 76% complete...
Removing image: 77% complete...
Removing image: 78% complete...
Removing image: 79% complete...
Removing image: 80% complete...
Removing image: 81% complete...
Removing image: 82% complete...
Removing image: 83% complete...
Removing image: 84% complete...
Removing image: 85% complete...
Removing image: 86% complete...
Removing image: 87% complete...
Removing image: 88% complete...
Removing image: 89% complete...
Removing image: 90% complete...
Removing image: 91% complete...
Removing image: 92% complete...
Removing image: 93% complete...
Removing image: 94% complete...
Removing image: 95% complete...
Removing image: 96% complete...
Removing image: 97% complete...
Removing image: 98% complete...
Removing image: 99% complete...
Removing image: 100% complete...done.
2024-05-17T15:07:38.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:38.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_present cluster1 mirror test deleted
2024-05-17T15:07:38.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=deleted
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image_id=
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n ''
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ get_image_id cluster1 mirror test
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local pool=mirror
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local image=test
2024-05-17T15:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster=cluster1 info mirror/test
2024-05-17T15:07:38.937 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
2024-05-17T15:07:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: error opening image test: (2) No such file or directory
2024-05-17T15:07:38.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image_id=
2024-05-17T15:07:38.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32
2024-05-17T15:07:38.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.1
2024-05-17T15:07:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_present cluster1 mirror test deleted ''
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=test
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=deleted
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image_id=
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=deleted
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_image_id
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ get_image_id cluster1 mirror test
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local pool=mirror
2024-05-17T15:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local image=test
2024-05-17T15:07:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster=cluster1 info mirror/test
2024-05-17T15:07:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
2024-05-17T15:07:39.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: error opening image test: (2) No such file or directory
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_image_id=
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test -n ''
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test deleted = deleted
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ testlog 'TEST: create many images'
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ tee -a /tmp/tmp.mfaeqiv6jj/rbd-mirror.test.log
2024-05-17T15:07:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ date '+%F %T'
2024-05-17T15:07:39.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo 2024-05-17 15:07:39 TEST: create many images
2024-05-17T15:07:39.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-17 15:07:39 TEST: create many images
2024-05-17T15:07:39.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ snap_name=snap
2024-05-17T15:07:39.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 50
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_1
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:07:39.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_1 128M
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_1
2024-05-17T15:07:39.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:07:39.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_1 snapshot
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:07:39.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot
2024-05-17T15:07:39.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:07:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_1
2024-05-17T15:07:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_1':
2024-05-17T15:07:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:07:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:07:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 126bcec8fdcf
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.126bcec8fdcf
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:07:39 2024
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:07:39 2024
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:07:39 2024
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:07:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 99506823-3ae7-4ad4-91fc-3c5dfa1d75a2
2024-05-17T15:07:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:07:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:07:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_1
2024-05-17T15:07:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:07:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:07:39.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:07:39.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:07:39.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:07:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 38939648 bytes
2024-05-17T15:07:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 48458752 bytes
2024-05-17T15:07:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 133281792 bytes
2024-05-17T15:07:40.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 188669952 bytes
2024-05-17T15:07:40.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 195220480 bytes
2024-05-17T15:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 218457088 bytes
2024-05-17T15:07:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 248141824 bytes
2024-05-17T15:07:41.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 290535424 bytes
2024-05-17T15:07:41.180 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 313137152 bytes
2024-05-17T15:07:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 354157568 bytes
2024-05-17T15:07:41.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 407289856 bytes
2024-05-17T15:07:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 461986816 bytes
2024-05-17T15:07:41.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 526738432 bytes
2024-05-17T15:07:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 610419712 bytes
2024-05-17T15:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 642225152 bytes
2024-05-17T15:07:42.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 694935552 bytes
2024-05-17T15:07:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 735585280 bytes
2024-05-17T15:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 817582080 bytes
2024-05-17T15:07:43.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 896566272 bytes
2024-05-17T15:07:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 934251520 bytes
2024-05-17T15:07:43.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 953501696 bytes
2024-05-17T15:07:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 991378432 bytes
2024-05-17T15:07:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1053086720 bytes
2024-05-17T15:07:44.157 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1117636608 bytes
2024-05-17T15:07:44.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1148622848 bytes
2024-05-17T15:07:44.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1153783808 bytes
2024-05-17T15:07:44.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1174687744 bytes
2024-05-17T15:07:44.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1274508288 bytes
2024-05-17T15:07:45.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1393711104 bytes
2024-05-17T15:07:45.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1439148032 bytes
2024-05-17T15:07:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1471490048 bytes
2024-05-17T15:07:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1535363072 bytes
2024-05-17T15:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1595655168 bytes
2024-05-17T15:07:46.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1607260160 bytes
2024-05-17T15:07:46.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1645900800 bytes
2024-05-17T15:07:46.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1663255552 bytes
2024-05-17T15:07:46.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1706420224 bytes
2024-05-17T15:07:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1774348288 bytes
2024-05-17T15:07:46.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1799826432 bytes
2024-05-17T15:07:47.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1877801984 bytes
2024-05-17T15:07:47.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1918714880 bytes
2024-05-17T15:07:47.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1929483264 bytes
2024-05-17T15:07:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1949526016 bytes
2024-05-17T15:07:47.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1965006848 bytes
2024-05-17T15:07:47.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1985339392 bytes
2024-05-17T15:07:47.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2020827136 bytes
2024-05-17T15:07:48.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2049107968 bytes
2024-05-17T15:07:48.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2131584000 bytes
2024-05-17T15:07:48.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2149790720 bytes
2024-05-17T15:07:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2176181248 bytes
2024-05-17T15:07:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2206232576 bytes
2024-05-17T15:07:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2241710080 bytes
2024-05-17T15:07:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2263368704 bytes
2024-05-17T15:07:49.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2400954368 bytes
2024-05-17T15:07:49.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2416729088 bytes
2024-05-17T15:07:49.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2448387072 bytes
2024-05-17T15:07:49.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2521064448 bytes
2024-05-17T15:07:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2568982528 bytes
2024-05-17T15:07:50.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2683469824 bytes
2024-05-17T15:07:50.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2703783936 bytes
2024-05-17T15:07:50.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2769994752 bytes
2024-05-17T15:07:50.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2786204672 bytes
2024-05-17T15:07:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2827030528 bytes
2024-05-17T15:07:51.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2934369280 bytes
2024-05-17T15:07:51.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2963829760 bytes
2024-05-17T15:07:51.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2971998208 bytes
2024-05-17T15:07:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2992701440 bytes
2024-05-17T15:07:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 3060062208 bytes
2024-05-17T15:07:52.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 3085760512 bytes
2024-05-17T15:07:52.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 3109049344 bytes
2024-05-17T15:07:52.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3122190336 bytes
2024-05-17T15:07:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3209900032 bytes
2024-05-17T15:07:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 3211328512 bytes
2024-05-17T15:07:52.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 3262892032 bytes
2024-05-17T15:07:53.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 3281092608 bytes
2024-05-17T15:07:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3321781248 bytes
2024-05-17T15:07:53.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7700 IOs, 3363560448 bytes
2024-05-17T15:07:53.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7800 IOs, 3430653952 bytes
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_2
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_2 128M
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:07:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:07:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:07:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:07:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_2
2024-05-17T15:07:53.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:07:53.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_2 snapshot
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:07:53.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot
2024-05-17T15:07:54.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:07:54.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_2
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_2':
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1277f412d247
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1277f412d247
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:07:53 2024
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:07:53 2024
2024-05-17T15:07:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:07:53 2024
2024-05-17T15:07:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:07:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:07:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 4ca271f9-b125-44e2-b0c0-f98df8231840
2024-05-17T15:07:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:07:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:07:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_2
2024-05-17T15:07:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:07:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:07:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:07:54.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:07:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:07:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:07:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:07:54.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 52564992 bytes
2024-05-17T15:07:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 69768192 bytes
2024-05-17T15:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 99629056 bytes
2024-05-17T15:07:55.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 137181184 bytes
2024-05-17T15:07:55.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 150685696 bytes
2024-05-17T15:07:55.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 151307264 bytes
2024-05-17T15:07:55.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 173676544 bytes
2024-05-17T15:07:55.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 188215296 bytes
2024-05-17T15:07:55.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 205894656 bytes
2024-05-17T15:07:55.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 233867264 bytes
2024-05-17T15:07:56.014 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5026609914127167) > weight (0.25)
2024-05-17T15:07:56.014 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.44673805214836704) > weight (0.25)
2024-05-17T15:07:56.014 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T15:07:56.015 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T15:07:56.015 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons
2024-05-17T15:07:56.015 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T15:07:56.014+0000 7fa8a718b640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 34638) UID: 1000
2024-05-17T15:07:56.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 256151552 bytes
2024-05-17T15:07:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 290564096 bytes
2024-05-17T15:07:56.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 291183616 bytes
2024-05-17T15:07:56.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 294626304 bytes
2024-05-17T15:07:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 349708288 bytes
2024-05-17T15:07:56.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 371589120 bytes
2024-05-17T15:07:56.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 417078272 bytes
2024-05-17T15:07:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 434091008 bytes
2024-05-17T15:07:57.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 486505472 bytes
2024-05-17T15:07:57.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 534311936 bytes
2024-05-17T15:07:57.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 608309248 bytes
2024-05-17T15:07:57.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 660307968 bytes
2024-05-17T15:07:57.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 677211136 bytes
2024-05-17T15:07:58.016 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T15:07:58.016 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:07:58.016 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T15:07:58.016 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T15:07:58.017 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T15:07:58.019 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T15:07:58.019 INFO:tasks.rbd_mirror_thrash:waiting for 67 secs before thrashing
2024-05-17T15:07:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 692299776 bytes
2024-05-17T15:07:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 733958144 bytes
2024-05-17T15:07:58.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 746293248 bytes
2024-05-17T15:07:58.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 809910272 bytes
2024-05-17T15:07:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 900263936 bytes
2024-05-17T15:07:59.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 947381248 bytes
2024-05-17T15:07:59.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 947985408 bytes
2024-05-17T15:07:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1000768512 bytes
2024-05-17T15:07:59.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1026753536 bytes
2024-05-17T15:07:59.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1066079232 bytes
2024-05-17T15:08:00.040 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1088002048 bytes
2024-05-17T15:08:00.259 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1150326784 bytes
2024-05-17T15:08:00.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1200566272 bytes
2024-05-17T15:08:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1267915776 bytes
2024-05-17T15:08:00.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1275524096 bytes
2024-05-17T15:08:01.048 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1344525312 bytes
2024-05-17T15:08:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1437067264 bytes
2024-05-17T15:08:01.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1468052480 bytes
2024-05-17T15:08:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1498829824 bytes
2024-05-17T15:08:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1522985984 bytes
2024-05-17T15:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1577868288 bytes
2024-05-17T15:08:02.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1611352064 bytes
2024-05-17T15:08:02.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1682369536 bytes
2024-05-17T15:08:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1765507072 bytes
2024-05-17T15:08:02.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1801216000 bytes
2024-05-17T15:08:02.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1859609600 bytes
2024-05-17T15:08:03.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 1911355392 bytes
2024-05-17T15:08:03.426 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 1991635968 bytes
2024-05-17T15:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2060502016 bytes
2024-05-17T15:08:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2117666816 bytes
2024-05-17T15:08:04.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2175577088 bytes
2024-05-17T15:08:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2213684224 bytes
2024-05-17T15:08:04.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2316014592 bytes
2024-05-17T15:08:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2380224512 bytes
2024-05-17T15:08:04.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2419525632 bytes
2024-05-17T15:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2483620864 bytes
2024-05-17T15:08:05.335 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2505572352 bytes
2024-05-17T15:08:05.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2563687424 bytes
2024-05-17T15:08:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2631743488 bytes
2024-05-17T15:08:05.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2659292160 bytes
2024-05-17T15:08:06.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2711921664 bytes
2024-05-17T15:08:06.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2712538112 bytes
2024-05-17T15:08:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2769588224 bytes
2024-05-17T15:08:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2798649344 bytes
2024-05-17T15:08:06.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2817262592 bytes
2024-05-17T15:08:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2877650944 bytes
2024-05-17T15:08:07.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2888573952 bytes
2024-05-17T15:08:07.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 2938506240 bytes
2024-05-17T15:08:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 2968325120 bytes
2024-05-17T15:08:07.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 2970398720 bytes
2024-05-17T15:08:07.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 2993275904 bytes
2024-05-17T15:08:07.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 3032285184 bytes
2024-05-17T15:08:08.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3080694784 bytes
2024-05-17T15:08:08.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7700 IOs, 3138435072 bytes
2024-05-17T15:08:08.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7800 IOs, 3175691264 bytes
2024-05-17T15:08:08.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7900 IOs, 3196609536 bytes
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_3
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M
2024-05-17T15:08:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_3 128M
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_3
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_3 snapshot
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot
2024-05-17T15:08:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:08:09.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_3
2024-05-17T15:08:09.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_3':
2024-05-17T15:08:09.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 128c2b94e110
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.128c2b94e110
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:08 2024
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:08 2024
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:08 2024
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:08:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:08:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 5814ab80-a23a-4a60-ba3c-ddb2e8964a09
2024-05-17T15:08:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_3
2024-05-17T15:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:08:09.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:08:09.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:08:09.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:08:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 58177536 bytes
2024-05-17T15:08:10.136 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 148258816 bytes
2024-05-17T15:08:10.348 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 208036864 bytes
2024-05-17T15:08:10.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 229884928 bytes
2024-05-17T15:08:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 287452160 bytes
2024-05-17T15:08:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 317880320 bytes
2024-05-17T15:08:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 323604480 bytes
2024-05-17T15:08:11.158 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 388051968 bytes
2024-05-17T15:08:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 457764864 bytes
2024-05-17T15:08:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 477787136 bytes
2024-05-17T15:08:11.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 533157888 bytes
2024-05-17T15:08:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 636472320 bytes
2024-05-17T15:08:12.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 672563200 bytes
2024-05-17T15:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 693392384 bytes
2024-05-17T15:08:12.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 716775424 bytes
2024-05-17T15:08:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 741188608 bytes
2024-05-17T15:08:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 742526976 bytes
2024-05-17T15:08:12.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 812834816 bytes
2024-05-17T15:08:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 839214080 bytes
2024-05-17T15:08:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 880565248 bytes
2024-05-17T15:08:13.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 923473920 bytes
2024-05-17T15:08:13.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 968099840 bytes
2024-05-17T15:08:13.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 971669504 bytes
2024-05-17T15:08:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 999977984 bytes
2024-05-17T15:08:14.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1047848960 bytes
2024-05-17T15:08:14.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1092909056 bytes
2024-05-17T15:08:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1109405696 bytes
2024-05-17T15:08:14.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1147650048 bytes
2024-05-17T15:08:14.753 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1197021184 bytes
2024-05-17T15:08:14.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1264496640 bytes
2024-05-17T15:08:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1292110848 bytes
2024-05-17T15:08:15.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1350949888 bytes
2024-05-17T15:08:15.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1416845312 bytes
2024-05-17T15:08:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1457091584 bytes
2024-05-17T15:08:15.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1513232384 bytes
2024-05-17T15:08:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1524892672 bytes
2024-05-17T15:08:16.180 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1555959808 bytes
2024-05-17T15:08:16.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1627833344 bytes
2024-05-17T15:08:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1668691968 bytes
2024-05-17T15:08:16.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1701348352 bytes
2024-05-17T15:08:16.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1730273280 bytes
2024-05-17T15:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1803494400 bytes
2024-05-17T15:08:17.234 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1811512320 bytes
2024-05-17T15:08:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1819089920 bytes
2024-05-17T15:08:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1884441600 bytes
2024-05-17T15:08:17.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1930058752 bytes
2024-05-17T15:08:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2016345088 bytes
2024-05-17T15:08:18.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2063329280 bytes
2024-05-17T15:08:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2102735872 bytes
2024-05-17T15:08:18.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2137942016 bytes
2024-05-17T15:08:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2153622528 bytes
2024-05-17T15:08:18.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2240427008 bytes
2024-05-17T15:08:19.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2254224384 bytes
2024-05-17T15:08:19.159 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2312357888 bytes
2024-05-17T15:08:19.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2394839040 bytes
2024-05-17T15:08:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2416126976 bytes
2024-05-17T15:08:19.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2430316544 bytes
2024-05-17T15:08:19.849 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2466031616 bytes
2024-05-17T15:08:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2492918784 bytes
2024-05-17T15:08:20.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2541202432 bytes
2024-05-17T15:08:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2541833216 bytes
2024-05-17T15:08:20.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2630197248 bytes
2024-05-17T15:08:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2691736576 bytes
2024-05-17T15:08:20.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2747575296 bytes
2024-05-17T15:08:21.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2769158144 bytes
2024-05-17T15:08:21.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2769806336 bytes
2024-05-17T15:08:21.392 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2831201280 bytes
2024-05-17T15:08:21.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2873170944 bytes
2024-05-17T15:08:21.719 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2887487488 bytes
2024-05-17T15:08:21.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2895417344 bytes
2024-05-17T15:08:21.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 2903403520 bytes
2024-05-17T15:08:22.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 2966946816 bytes
2024-05-17T15:08:22.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 3054542848 bytes
2024-05-17T15:08:22.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 3147693056 bytes
2024-05-17T15:08:22.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 3190227968 bytes
2024-05-17T15:08:23.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3223564288 bytes
2024-05-17T15:08:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7700 IOs, 3269696512 bytes
2024-05-17T15:08:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7800 IOs, 3270292480 bytes
2024-05-17T15:08:23.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7900 IOs, 3300672512 bytes
2024-05-17T15:08:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_4
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_4 128M
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:08:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:08:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:08:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_4
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_4 snapshot
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot
2024-05-17T15:08:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:08:24.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_4
2024-05-17T15:08:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_4':
2024-05-17T15:08:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:08:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:08:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:08:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1298b6543929
2024-05-17T15:08:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1298b6543929
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:23 2024
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:23 2024
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:23 2024
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 5c3f1454-afcf-43a9-8f75-0f06d2c9f10d
2024-05-17T15:08:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:08:24.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:08:24.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_4
2024-05-17T15:08:24.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:24.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:24.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:08:24.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:08:24.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:08:24.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:08:24.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:08:24.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 19388416 bytes
2024-05-17T15:08:25.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 52221952 bytes
2024-05-17T15:08:25.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 74407936 bytes
2024-05-17T15:08:25.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 122542080 bytes
2024-05-17T15:08:25.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 137763840 bytes
2024-05-17T15:08:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 159426560 bytes
2024-05-17T15:08:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 230014976 bytes
2024-05-17T15:08:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 242119680 bytes
2024-05-17T15:08:26.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 262142976 bytes
2024-05-17T15:08:26.207 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 265671680 bytes
2024-05-17T15:08:26.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 340255744 bytes
2024-05-17T15:08:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 421260288 bytes
2024-05-17T15:08:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 446695424 bytes
2024-05-17T15:08:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 527169536 bytes
2024-05-17T15:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 586208256 bytes
2024-05-17T15:08:27.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 611980288 bytes
2024-05-17T15:08:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 645962752 bytes
2024-05-17T15:08:27.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 677026816 bytes
2024-05-17T15:08:28.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 744684544 bytes
2024-05-17T15:08:28.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 832286720 bytes
2024-05-17T15:08:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 886955008 bytes
2024-05-17T15:08:28.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 910110720 bytes
2024-05-17T15:08:29.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 982681600 bytes
2024-05-17T15:08:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1012253696 bytes
2024-05-17T15:08:29.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1085794304 bytes
2024-05-17T15:08:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1136406528 bytes
2024-05-17T15:08:29.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1161754624 bytes
2024-05-17T15:08:30.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1203492864 bytes
2024-05-17T15:08:30.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1252649984 bytes
2024-05-17T15:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1276986368 bytes
2024-05-17T15:08:30.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1277629440 bytes
2024-05-17T15:08:30.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1278245888 bytes
2024-05-17T15:08:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1324546048 bytes
2024-05-17T15:08:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1365002240 bytes
2024-05-17T15:08:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1410360320 bytes
2024-05-17T15:08:31.278 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1460352000 bytes
2024-05-17T15:08:31.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1494651904 bytes
2024-05-17T15:08:31.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1511127040 bytes
2024-05-17T15:08:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:08:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:08:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_5
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_5 128M
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:31.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_5
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_5 snapshot
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:08:31.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot
2024-05-17T15:08:32.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:08:32.250 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_5
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_5':
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12a4f9deb8c9
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12a4f9deb8c9
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:08:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:31 2024
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:31 2024
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:31 2024
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 6a7fbf74-3f03-4071-9e4a-10ba83516c52
2024-05-17T15:08:32.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:08:32.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:08:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_5
2024-05-17T15:08:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:08:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:08:32.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:08:32.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:08:32.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:08:32.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 64909312 bytes
2024-05-17T15:08:32.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 137060352 bytes
2024-05-17T15:08:32.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 209131520 bytes
2024-05-17T15:08:33.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 295528448 bytes
2024-05-17T15:08:33.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 324675584 bytes
2024-05-17T15:08:33.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 407544832 bytes
2024-05-17T15:08:33.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 432415744 bytes
2024-05-17T15:08:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 508198912 bytes
2024-05-17T15:08:34.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 535677952 bytes
2024-05-17T15:08:34.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 586485760 bytes
2024-05-17T15:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 620819456 bytes
2024-05-17T15:08:34.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 639528960 bytes
2024-05-17T15:08:34.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 669452288 bytes
2024-05-17T15:08:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 699511808 bytes
2024-05-17T15:08:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 787244032 bytes
2024-05-17T15:08:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 851185664 bytes
2024-05-17T15:08:35.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 864718848 bytes
2024-05-17T15:08:35.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 897805312 bytes
2024-05-17T15:08:35.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 930814976 bytes
2024-05-17T15:08:36.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 941849600 bytes
2024-05-17T15:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 997440512 bytes
2024-05-17T15:08:36.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1042059264 bytes
2024-05-17T15:08:36.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1060716544 bytes
2024-05-17T15:08:36.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1121319936 bytes
2024-05-17T15:08:36.844 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1139412992 bytes
2024-05-17T15:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1161201664 bytes
2024-05-17T15:08:37.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1232849920 bytes
2024-05-17T15:08:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1233467392 bytes
2024-05-17T15:08:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1264066560 bytes
2024-05-17T15:08:37.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1279776768 bytes
2024-05-17T15:08:37.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1371547648 bytes
2024-05-17T15:08:38.013 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1440168960 bytes
2024-05-17T15:08:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1481704448 bytes
2024-05-17T15:08:38.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1503545344 bytes
2024-05-17T15:08:38.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1571800064 bytes
2024-05-17T15:08:38.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1579173888 bytes
2024-05-17T15:08:38.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1596007424 bytes
2024-05-17T15:08:39.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1661266944 bytes
2024-05-17T15:08:39.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1661884416 bytes
2024-05-17T15:08:39.330 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1733212160 bytes
2024-05-17T15:08:39.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1791883264 bytes
2024-05-17T15:08:39.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1829959680 bytes
2024-05-17T15:08:39.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1830572032 bytes
2024-05-17T15:08:40.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1924858880 bytes
2024-05-17T15:08:40.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1954446336 bytes
2024-05-17T15:08:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2006802432 bytes
2024-05-17T15:08:40.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2053669888 bytes
2024-05-17T15:08:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2128565248 bytes
2024-05-17T15:08:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2170223616 bytes
2024-05-17T15:08:41.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2216838144 bytes
2024-05-17T15:08:41.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2297751552 bytes
2024-05-17T15:08:41.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2334492672 bytes
2024-05-17T15:08:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2351948800 bytes
2024-05-17T15:08:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2399752192 bytes
2024-05-17T15:08:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2458914816 bytes
2024-05-17T15:08:42.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2517473280 bytes
2024-05-17T15:08:42.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2564350976 bytes
2024-05-17T15:08:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2620359680 bytes
2024-05-17T15:08:43.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2633425920 bytes
2024-05-17T15:08:43.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2657148928 bytes
2024-05-17T15:08:43.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2679884800 bytes
2024-05-17T15:08:43.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2697984000 bytes
2024-05-17T15:08:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2774441984 bytes
2024-05-17T15:08:43.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2782969856 bytes
2024-05-17T15:08:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2787195904 bytes
2024-05-17T15:08:44.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2808437760 bytes
2024-05-17T15:08:44.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2849589248 bytes
2024-05-17T15:08:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2869866496 bytes
2024-05-17T15:08:44.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2911537152 bytes
2024-05-17T15:08:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2948042752 bytes
2024-05-17T15:08:44.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3005836288 bytes
2024-05-17T15:08:45.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3021291520 bytes
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_6
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M
2024-05-17T15:08:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_6 128M
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:08:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:08:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:08:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:08:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_6
2024-05-17T15:08:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:08:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:08:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_6 snapshot
2024-05-17T15:08:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:08:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot
2024-05-17T15:08:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:08:46.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_6
2024-05-17T15:08:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_6':
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12b0f3ddb5ec
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12b0f3ddb5ec
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:45 2024
2024-05-17T15:08:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:45 2024
2024-05-17T15:08:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:45 2024
2024-05-17T15:08:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:08:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:08:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7aeb365d-f128-4cac-a752-c8e9bc4d1961
2024-05-17T15:08:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:08:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:08:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_6
2024-05-17T15:08:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:08:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:08:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=10
2024-05-17T15:08:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:08:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:08:46.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 66105344 bytes
2024-05-17T15:08:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 99408896 bytes
2024-05-17T15:08:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 129091584 bytes
2024-05-17T15:08:47.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 190109696 bytes
2024-05-17T15:08:47.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 235039744 bytes
2024-05-17T15:08:47.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 281027584 bytes
2024-05-17T15:08:47.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 302837760 bytes
2024-05-17T15:08:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 303458304 bytes
2024-05-17T15:08:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 339484672 bytes
2024-05-17T15:08:48.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 371554304 bytes
2024-05-17T15:08:48.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 413408256 bytes
2024-05-17T15:08:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 435955712 bytes
2024-05-17T15:08:48.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 442567680 bytes
2024-05-17T15:08:48.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 495243264 bytes
2024-05-17T15:08:49.133 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 610971648 bytes
2024-05-17T15:08:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 639175680 bytes
2024-05-17T15:08:49.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 643176448 bytes
2024-05-17T15:08:49.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 658902016 bytes
2024-05-17T15:08:49.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 678476800 bytes
2024-05-17T15:08:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 696955904 bytes
2024-05-17T15:08:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 708727808 bytes
2024-05-17T15:08:50.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 728888320 bytes
2024-05-17T15:08:50.159 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 740782080 bytes
2024-05-17T15:08:50.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 816525312 bytes
2024-05-17T15:08:50.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 873374720 bytes
2024-05-17T15:08:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 933866496 bytes
2024-05-17T15:08:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 980188160 bytes
2024-05-17T15:08:51.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1016658944 bytes
2024-05-17T15:08:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1086470144 bytes
2024-05-17T15:08:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1098368000 bytes
2024-05-17T15:08:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1098986496 bytes
2024-05-17T15:08:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1131830272 bytes
2024-05-17T15:08:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1132441600 bytes
2024-05-17T15:08:52.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1186573312 bytes
2024-05-17T15:08:52.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1211302912 bytes
2024-05-17T15:08:52.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1257268224 bytes
2024-05-17T15:08:52.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1265619968 bytes
2024-05-17T15:08:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1279321088 bytes
2024-05-17T15:08:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1339195392 bytes
2024-05-17T15:08:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1357332480 bytes
2024-05-17T15:08:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1370954752 bytes
2024-05-17T15:08:53.350 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1438819328 bytes
2024-05-17T15:08:53.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1479780352 bytes
2024-05-17T15:08:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1519080448 bytes
2024-05-17T15:08:53.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1581718528 bytes
2024-05-17T15:08:54.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1628609536 bytes
2024-05-17T15:08:54.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1677296640 bytes
2024-05-17T15:08:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1677921280 bytes
2024-05-17T15:08:54.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1751781376 bytes
2024-05-17T15:08:54.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 1779991552 bytes
2024-05-17T15:08:54.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 1780604928 bytes
2024-05-17T15:08:55.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 1824048128 bytes
2024-05-17T15:08:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 1878307840 bytes
2024-05-17T15:08:55.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 1883020288 bytes
2024-05-17T15:08:55.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 1899196416 bytes
2024-05-17T15:08:55.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 1954579456 bytes
2024-05-17T15:08:55.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2003416064 bytes
2024-05-17T15:08:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2018295808 bytes
2024-05-17T15:08:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2099535872 bytes
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_7
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M
2024-05-17T15:08:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_7 128M
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:08:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:08:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:08:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:08:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:08:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_7
2024-05-17T15:08:56.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_7 snapshot
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:08:56.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot
2024-05-17T15:08:57.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:08:57.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_7
2024-05-17T15:08:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_7':
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12bcf7ca7fba
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12bcf7ca7fba
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:56 2024
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:56 2024
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:56 2024
2024-05-17T15:08:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:08:57.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:08:57.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 26533277-dd10-458b-9fe7-f5ca55df003d
2024-05-17T15:08:57.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:08:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:08:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_7
2024-05-17T15:08:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:08:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:08:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:08:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:08:57.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:08:57.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:08:57.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:08:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 82513920 bytes
2024-05-17T15:08:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 120273920 bytes
2024-05-17T15:08:58.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 196402176 bytes
2024-05-17T15:08:58.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 255060992 bytes
2024-05-17T15:08:58.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 357287936 bytes
2024-05-17T15:08:58.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 447221760 bytes
2024-05-17T15:08:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 474493952 bytes
2024-05-17T15:08:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 499202048 bytes
2024-05-17T15:08:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 556062720 bytes
2024-05-17T15:08:59.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 621261824 bytes
2024-05-17T15:08:59.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 663839744 bytes
2024-05-17T15:09:00.157 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 733999104 bytes
2024-05-17T15:09:00.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 761703424 bytes
2024-05-17T15:09:00.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 790795264 bytes
2024-05-17T15:09:00.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 822520832 bytes
2024-05-17T15:09:00.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 900697088 bytes
2024-05-17T15:09:01.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 911488000 bytes
2024-05-17T15:09:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 982892544 bytes
2024-05-17T15:09:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 1027019776 bytes
2024-05-17T15:09:01.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 1027641344 bytes
2024-05-17T15:09:01.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1028239360 bytes
2024-05-17T15:09:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1028838400 bytes
2024-05-17T15:09:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1051052032 bytes
2024-05-17T15:09:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1094318080 bytes
2024-05-17T15:09:02.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1120859136 bytes
2024-05-17T15:09:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1121446912 bytes
2024-05-17T15:09:02.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1198888960 bytes
2024-05-17T15:09:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1242939392 bytes
2024-05-17T15:09:02.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1252710400 bytes
2024-05-17T15:09:03.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1253321728 bytes
2024-05-17T15:09:03.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1295961088 bytes
2024-05-17T15:09:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1315725312 bytes
2024-05-17T15:09:03.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1327512576 bytes
2024-05-17T15:09:03.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1368413184 bytes
2024-05-17T15:09:04.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1442241536 bytes
2024-05-17T15:09:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1507769344 bytes
2024-05-17T15:09:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1520130048 bytes
2024-05-17T15:09:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1550582784 bytes
2024-05-17T15:09:04.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1634562048 bytes
2024-05-17T15:09:05.020 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8257319237271559) > weight (0.25)
2024-05-17T15:09:05.020 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6662781201232979) > weight (0.25)
2024-05-17T15:09:05.020 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8408282056054922) > weight (0.25)
2024-05-17T15:09:05.021 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6720237491967967) > weight (0.25)
2024-05-17T15:09:05.021 INFO:tasks.rbd_mirror_thrash:waiting for 78 secs before thrashing
2024-05-17T15:09:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1697805312 bytes
2024-05-17T15:09:05.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1722417152 bytes
2024-05-17T15:09:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1772191744 bytes
2024-05-17T15:09:05.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1846165504 bytes
2024-05-17T15:09:06.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1851157504 bytes
2024-05-17T15:09:06.152 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1878718464 bytes
2024-05-17T15:09:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1921428480 bytes
2024-05-17T15:09:06.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1965429760 bytes
2024-05-17T15:09:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2021861376 bytes
2024-05-17T15:09:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2022445056 bytes
2024-05-17T15:09:07.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2049876992 bytes
2024-05-17T15:09:07.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2120038400 bytes
2024-05-17T15:09:07.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2145063936 bytes
2024-05-17T15:09:07.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2174027776 bytes
2024-05-17T15:09:07.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2218797056 bytes
2024-05-17T15:09:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2264957952 bytes
2024-05-17T15:09:08.097 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2302030848 bytes
2024-05-17T15:09:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2347786240 bytes
2024-05-17T15:09:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2411609088 bytes
2024-05-17T15:09:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2462963712 bytes
2024-05-17T15:09:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2520785920 bytes
2024-05-17T15:09:09.157 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2583248896 bytes
2024-05-17T15:09:09.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2630463488 bytes
2024-05-17T15:09:09.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2665756672 bytes
2024-05-17T15:09:09.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2701804544 bytes
2024-05-17T15:09:09.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2793348096 bytes
2024-05-17T15:09:10.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2813981696 bytes
2024-05-17T15:09:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2861663232 bytes
2024-05-17T15:09:10.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2896197632 bytes
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_8
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_8 128M
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:09:10.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:09:10.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:10.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_8
2024-05-17T15:09:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:09:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:09:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_8 snapshot
2024-05-17T15:09:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:09:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot
2024-05-17T15:09:11.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:09:11.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_8
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_8':
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12c8bc3d2209
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12c8bc3d2209
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:09:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:10 2024
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:10 2024
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:10 2024
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: f76cd062-3e00-458d-a951-27acd55808b3
2024-05-17T15:09:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:09:11.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:09:11.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_8
2024-05-17T15:09:11.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:11.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:11.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:09:11.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:09:11.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:09:11.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:09:11.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:09:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 120809472 bytes
2024-05-17T15:09:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 165678080 bytes
2024-05-17T15:09:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 172337152 bytes
2024-05-17T15:09:12.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 254410752 bytes
2024-05-17T15:09:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 276563968 bytes
2024-05-17T15:09:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 330171392 bytes
2024-05-17T15:09:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 401532928 bytes
2024-05-17T15:09:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 446225408 bytes
2024-05-17T15:09:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 481750016 bytes
2024-05-17T15:09:13.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 500018176 bytes
2024-05-17T15:09:13.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 556886016 bytes
2024-05-17T15:09:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 610385920 bytes
2024-05-17T15:09:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 668169216 bytes
2024-05-17T15:09:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 709873664 bytes
2024-05-17T15:09:14.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 811027456 bytes
2024-05-17T15:09:14.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 850184192 bytes
2024-05-17T15:09:14.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 866695168 bytes
2024-05-17T15:09:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 924702720 bytes
2024-05-17T15:09:15.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 944140288 bytes
2024-05-17T15:09:15.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 994600960 bytes
2024-05-17T15:09:15.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1046180864 bytes
2024-05-17T15:09:15.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1056452608 bytes
2024-05-17T15:09:15.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1078658048 bytes
2024-05-17T15:09:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1104758784 bytes
2024-05-17T15:09:16.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1120706560 bytes
2024-05-17T15:09:16.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1143880704 bytes
2024-05-17T15:09:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1206877184 bytes
2024-05-17T15:09:16.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1314063360 bytes
2024-05-17T15:09:17.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1384317952 bytes
2024-05-17T15:09:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1391734784 bytes
2024-05-17T15:09:17.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1430012928 bytes
2024-05-17T15:09:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1446132736 bytes
2024-05-17T15:09:17.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1475109888 bytes
2024-05-17T15:09:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1514483712 bytes
2024-05-17T15:09:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1537069056 bytes
2024-05-17T15:09:17.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1549299712 bytes
2024-05-17T15:09:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1594067968 bytes
2024-05-17T15:09:18.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1620197376 bytes
2024-05-17T15:09:18.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1676312576 bytes
2024-05-17T15:09:18.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1717010432 bytes
2024-05-17T15:09:18.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1734416384 bytes
2024-05-17T15:09:18.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1760524288 bytes
2024-05-17T15:09:19.028 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7859639834219313) > weight (0.25)
2024-05-17T15:09:19.029 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4762471922102297) > weight (0.25)
2024-05-17T15:09:19.029 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.48871086557643195) > weight (0.25)
2024-05-17T15:09:19.029 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8418650061938311) > weight (0.25)
2024-05-17T15:09:19.029 INFO:tasks.rbd_mirror_thrash:waiting for 67 secs before thrashing
2024-05-17T15:09:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1773918208 bytes
2024-05-17T15:09:19.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1787503616 bytes
2024-05-17T15:09:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1824356352 bytes
2024-05-17T15:09:19.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1868028928 bytes
2024-05-17T15:09:19.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1948206080 bytes
2024-05-17T15:09:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1967736832 bytes
2024-05-17T15:09:20.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2004653056 bytes
2024-05-17T15:09:20.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2005250048 bytes
2024-05-17T15:09:20.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2052267008 bytes
2024-05-17T15:09:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2068946944 bytes
2024-05-17T15:09:20.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2117246976 bytes
2024-05-17T15:09:20.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2122805248 bytes
2024-05-17T15:09:20.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2127990784 bytes
2024-05-17T15:09:21.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2175649792 bytes
2024-05-17T15:09:21.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2202252288 bytes
2024-05-17T15:09:21.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2278021120 bytes
2024-05-17T15:09:21.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2332410880 bytes
2024-05-17T15:09:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2394028032 bytes
2024-05-17T15:09:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2394631168 bytes
2024-05-17T15:09:22.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2434952192 bytes
2024-05-17T15:09:22.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2435571712 bytes
2024-05-17T15:09:22.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2544043008 bytes
2024-05-17T15:09:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2637452288 bytes
2024-05-17T15:09:23.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2649515008 bytes
2024-05-17T15:09:23.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2694114304 bytes
2024-05-17T15:09:23.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2694717440 bytes
2024-05-17T15:09:23.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2770909184 bytes
2024-05-17T15:09:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2811075584 bytes
2024-05-17T15:09:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 2811683840 bytes
2024-05-17T15:09:23.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 2825780224 bytes
2024-05-17T15:09:24.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 2902252544 bytes
2024-05-17T15:09:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 2918039552 bytes
2024-05-17T15:09:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:09:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:09:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:09:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_9
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_9 128M
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_9
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_9 snapshot
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot
2024-05-17T15:09:25.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:09:25.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_9
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_9':
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12d48e26dfd8
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12d48e26dfd8
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:09:25.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:24 2024
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:24 2024
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:24 2024
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 128beb1d-d528-4574-85e6-dc2e3922fc06
2024-05-17T15:09:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:09:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:09:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_9
2024-05-17T15:09:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:09:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:09:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:09:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:09:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:09:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 84101120 bytes
2024-05-17T15:09:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 120400896 bytes
2024-05-17T15:09:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 164317184 bytes
2024-05-17T15:09:26.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 204551168 bytes
2024-05-17T15:09:26.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 259755008 bytes
2024-05-17T15:09:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 266648576 bytes
2024-05-17T15:09:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 381625344 bytes
2024-05-17T15:09:27.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 413713408 bytes
2024-05-17T15:09:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 414347264 bytes
2024-05-17T15:09:27.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 430235648 bytes
2024-05-17T15:09:27.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 464619520 bytes
2024-05-17T15:09:28.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 467089408 bytes
2024-05-17T15:09:28.314 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 540813312 bytes
2024-05-17T15:09:28.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 564497408 bytes
2024-05-17T15:09:28.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 615014400 bytes
2024-05-17T15:09:28.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 670226432 bytes
2024-05-17T15:09:29.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 704956416 bytes
2024-05-17T15:09:29.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 736833536 bytes
2024-05-17T15:09:29.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 790955008 bytes
2024-05-17T15:09:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 850954240 bytes
2024-05-17T15:09:29.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 892291072 bytes
2024-05-17T15:09:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 898856960 bytes
2024-05-17T15:09:30.040 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 921039872 bytes
2024-05-17T15:09:30.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1011130368 bytes
2024-05-17T15:09:30.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1039299584 bytes
2024-05-17T15:09:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1060930560 bytes
2024-05-17T15:09:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1108700160 bytes
2024-05-17T15:09:31.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1181248512 bytes
2024-05-17T15:09:31.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1266763776 bytes
2024-05-17T15:09:31.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1311900672 bytes
2024-05-17T15:09:31.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1329170432 bytes
2024-05-17T15:09:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1374163968 bytes
2024-05-17T15:09:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1418321920 bytes
2024-05-17T15:09:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1469065216 bytes
2024-05-17T15:09:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1498722304 bytes
2024-05-17T15:09:32.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1525909504 bytes
2024-05-17T15:09:32.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1570396160 bytes
2024-05-17T15:09:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1601733632 bytes
2024-05-17T15:09:33.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1646033920 bytes
2024-05-17T15:09:33.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1678009344 bytes
2024-05-17T15:09:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1735344128 bytes
2024-05-17T15:09:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1795235840 bytes
2024-05-17T15:09:34.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1854617600 bytes
2024-05-17T15:09:34.157 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1867351040 bytes
2024-05-17T15:09:34.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1927665664 bytes
2024-05-17T15:09:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1958571008 bytes
2024-05-17T15:09:34.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2003386368 bytes
2024-05-17T15:09:35.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2047820800 bytes
2024-05-17T15:09:35.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2065864704 bytes
2024-05-17T15:09:35.424 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2134423552 bytes
2024-05-17T15:09:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2135021568 bytes
2024-05-17T15:09:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2152366080 bytes
2024-05-17T15:09:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2172431360 bytes
2024-05-17T15:09:36.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2219020288 bytes
2024-05-17T15:09:36.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2300694528 bytes
2024-05-17T15:09:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2385256448 bytes
2024-05-17T15:09:36.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2438915072 bytes
2024-05-17T15:09:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2478489600 bytes
2024-05-17T15:09:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2544705536 bytes
2024-05-17T15:09:37.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2613282816 bytes
2024-05-17T15:09:37.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2613925888 bytes
2024-05-17T15:09:37.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2637569024 bytes
2024-05-17T15:09:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2676477952 bytes
2024-05-17T15:09:38.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2714384384 bytes
2024-05-17T15:09:38.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2739587072 bytes
2024-05-17T15:09:38.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2775397376 bytes
2024-05-17T15:09:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2830781440 bytes
2024-05-17T15:09:38.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2870579200 bytes
2024-05-17T15:09:38.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2907917312 bytes
2024-05-17T15:09:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 3011411968 bytes
2024-05-17T15:09:39.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3057622016 bytes
2024-05-17T15:09:39.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3102850048 bytes
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_10
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_10 128M
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:09:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:09:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:09:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:09:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_10
2024-05-17T15:09:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:09:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:09:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_10 snapshot
2024-05-17T15:09:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:09:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot
2024-05-17T15:09:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:09:40.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_10
2024-05-17T15:09:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_10':
2024-05-17T15:09:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:09:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:09:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:09:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12e1411ffd1f
2024-05-17T15:09:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12e1411ffd1f
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:39 2024
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:39 2024
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:39 2024
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 6a55ffff-b717-4cdd-9603-20c25dd3ae1c
2024-05-17T15:09:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:09:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:09:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_10
2024-05-17T15:09:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:09:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:09:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=10
2024-05-17T15:09:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:09:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:09:40.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 55120896 bytes
2024-05-17T15:09:41.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 112817152 bytes
2024-05-17T15:09:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 145709056 bytes
2024-05-17T15:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 175352832 bytes
2024-05-17T15:09:41.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 242694144 bytes
2024-05-17T15:09:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 302609408 bytes
2024-05-17T15:09:42.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 355225600 bytes
2024-05-17T15:09:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 377549824 bytes
2024-05-17T15:09:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 416264192 bytes
2024-05-17T15:09:42.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 436998144 bytes
2024-05-17T15:09:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 507980800 bytes
2024-05-17T15:09:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 544600064 bytes
2024-05-17T15:09:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 549937152 bytes
2024-05-17T15:09:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 659723264 bytes
2024-05-17T15:09:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 677707776 bytes
2024-05-17T15:09:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 682447872 bytes
2024-05-17T15:09:43.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 695792640 bytes
2024-05-17T15:09:43.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 711392256 bytes
2024-05-17T15:09:44.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 769170432 bytes
2024-05-17T15:09:44.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 799955968 bytes
2024-05-17T15:09:44.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 837671936 bytes
2024-05-17T15:09:44.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 838298624 bytes
2024-05-17T15:09:44.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 896260096 bytes
2024-05-17T15:09:44.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 921688064 bytes
2024-05-17T15:09:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 953159680 bytes
2024-05-17T15:09:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1019295744 bytes
2024-05-17T15:09:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1060235264 bytes
2024-05-17T15:09:45.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1091809280 bytes
2024-05-17T15:09:45.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1136834560 bytes
2024-05-17T15:09:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1266656256 bytes
2024-05-17T15:09:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1303054336 bytes
2024-05-17T15:09:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1342505984 bytes
2024-05-17T15:09:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1433615360 bytes
2024-05-17T15:09:46.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1517382656 bytes
2024-05-17T15:09:47.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1552698368 bytes
2024-05-17T15:09:47.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1587088384 bytes
2024-05-17T15:09:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1599588352 bytes
2024-05-17T15:09:47.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1618608128 bytes
2024-05-17T15:09:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1632190464 bytes
2024-05-17T15:09:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1717725184 bytes
2024-05-17T15:09:48.208 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1751942144 bytes
2024-05-17T15:09:48.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1762769920 bytes
2024-05-17T15:09:48.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1835010048 bytes
2024-05-17T15:09:48.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1861508096 bytes
2024-05-17T15:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1919223808 bytes
2024-05-17T15:09:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1965834240 bytes
2024-05-17T15:09:49.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2010677248 bytes
2024-05-17T15:09:49.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2011282432 bytes
2024-05-17T15:09:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2078166016 bytes
2024-05-17T15:09:49.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2094410752 bytes
2024-05-17T15:09:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2186863616 bytes
2024-05-17T15:09:50.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2196509696 bytes
2024-05-17T15:09:50.436 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2267708416 bytes
2024-05-17T15:09:50.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2334853120 bytes
2024-05-17T15:09:50.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_11
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_11 128M
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_11
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_11 snapshot
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:09:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot
2024-05-17T15:09:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:09:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_11
2024-05-17T15:09:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_11':
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12ed746872bf
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12ed746872bf
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:50 2024
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:50 2024
2024-05-17T15:09:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:50 2024
2024-05-17T15:09:50.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:09:50.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:09:50.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ccec9d36-68f0-4c69-b480-a47cd5e5df24
2024-05-17T15:09:50.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:09:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:09:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_11
2024-05-17T15:09:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:09:50.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:09:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:09:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:09:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:09:51.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 5910528 bytes
2024-05-17T15:09:51.312 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 63448064 bytes
2024-05-17T15:09:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 87145472 bytes
2024-05-17T15:09:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 109787136 bytes
2024-05-17T15:09:51.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 124334080 bytes
2024-05-17T15:09:51.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 137358336 bytes
2024-05-17T15:09:52.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 195393536 bytes
2024-05-17T15:09:52.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 216873984 bytes
2024-05-17T15:09:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 239414272 bytes
2024-05-17T15:09:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 246703104 bytes
2024-05-17T15:09:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 253604864 bytes
2024-05-17T15:09:52.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 301123584 bytes
2024-05-17T15:09:52.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 354632704 bytes
2024-05-17T15:09:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 386362368 bytes
2024-05-17T15:09:53.284 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 421704704 bytes
2024-05-17T15:09:53.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 479793152 bytes
2024-05-17T15:09:53.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 537209856 bytes
2024-05-17T15:09:53.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 590936064 bytes
2024-05-17T15:09:54.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 639044608 bytes
2024-05-17T15:09:54.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 671920128 bytes
2024-05-17T15:09:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 675056640 bytes
2024-05-17T15:09:54.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 721928192 bytes
2024-05-17T15:09:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 786325504 bytes
2024-05-17T15:09:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 804120576 bytes
2024-05-17T15:09:55.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 834519040 bytes
2024-05-17T15:09:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 836666368 bytes
2024-05-17T15:09:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 869799936 bytes
2024-05-17T15:09:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 902254592 bytes
2024-05-17T15:09:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 930793472 bytes
2024-05-17T15:09:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 950884352 bytes
2024-05-17T15:09:56.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 964015104 bytes
2024-05-17T15:09:56.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 984471552 bytes
2024-05-17T15:09:56.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1060207616 bytes
2024-05-17T15:09:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1063019520 bytes
2024-05-17T15:09:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1131090944 bytes
2024-05-17T15:09:57.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1170909184 bytes
2024-05-17T15:09:57.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1244129280 bytes
2024-05-17T15:09:57.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1325029376 bytes
2024-05-17T15:09:57.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1389105152 bytes
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_12
2024-05-17T15:09:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_12 128M
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:09:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_12
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_12 snapshot
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:09:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot
2024-05-17T15:09:58.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:09:58.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_12
2024-05-17T15:09:58.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_12':
2024-05-17T15:09:58.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12f9351a8618
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12f9351a8618
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:58 2024
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:58 2024
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:58 2024
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:09:58.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:09:58.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 915c9e6a-ae95-4135-a900-860abb54b53e
2024-05-17T15:09:58.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:09:58.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:09:58.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_12
2024-05-17T15:09:58.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:09:58.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:09:58.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:09:58.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:09:58.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=12
2024-05-17T15:09:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:09:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:09:58.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 1440768 bytes
2024-05-17T15:09:58.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 46605312 bytes
2024-05-17T15:09:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 90183680 bytes
2024-05-17T15:09:59.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 124763136 bytes
2024-05-17T15:09:59.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 214349824 bytes
2024-05-17T15:09:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 253878272 bytes
2024-05-17T15:09:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 259245056 bytes
2024-05-17T15:09:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 286770176 bytes
2024-05-17T15:10:00.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 372364288 bytes
2024-05-17T15:10:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 381493248 bytes
2024-05-17T15:10:00.389 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 446605312 bytes
2024-05-17T15:10:00.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 491329536 bytes
2024-05-17T15:10:00.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 512393216 bytes
2024-05-17T15:10:00.937 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 571557888 bytes
2024-05-17T15:10:01.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 653378560 bytes
2024-05-17T15:10:01.347 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 676316160 bytes
2024-05-17T15:10:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 728133632 bytes
2024-05-17T15:10:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 793367552 bytes
2024-05-17T15:10:02.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 836643840 bytes
2024-05-17T15:10:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 848863232 bytes
2024-05-17T15:10:02.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 907869184 bytes
2024-05-17T15:10:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 962321408 bytes
2024-05-17T15:10:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 994880512 bytes
2024-05-17T15:10:02.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1048094720 bytes
2024-05-17T15:10:03.235 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1148442624 bytes
2024-05-17T15:10:03.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1223345152 bytes
2024-05-17T15:10:03.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1291516928 bytes
2024-05-17T15:10:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1309203456 bytes
2024-05-17T15:10:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1309800448 bytes
2024-05-17T15:10:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1339586560 bytes
2024-05-17T15:10:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1368469504 bytes
2024-05-17T15:10:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1417583616 bytes
2024-05-17T15:10:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1419610112 bytes
2024-05-17T15:10:04.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1441681408 bytes
2024-05-17T15:10:04.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1495240704 bytes
2024-05-17T15:10:05.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1506591744 bytes
2024-05-17T15:10:05.314 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1582072832 bytes
2024-05-17T15:10:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1600169984 bytes
2024-05-17T15:10:05.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1693540352 bytes
2024-05-17T15:10:05.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1742757888 bytes
2024-05-17T15:10:06.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1775114240 bytes
2024-05-17T15:10:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1823003648 bytes
2024-05-17T15:10:06.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1855706112 bytes
2024-05-17T15:10:06.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1914465280 bytes
2024-05-17T15:10:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1962961920 bytes
2024-05-17T15:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2071130112 bytes
2024-05-17T15:10:07.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2090241024 bytes
2024-05-17T15:10:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2121005056 bytes
2024-05-17T15:10:07.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2182369280 bytes
2024-05-17T15:10:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2223422464 bytes
2024-05-17T15:10:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2275928064 bytes
2024-05-17T15:10:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2309485568 bytes
2024-05-17T15:10:08.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2370167808 bytes
2024-05-17T15:10:08.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2470935552 bytes
2024-05-17T15:10:09.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2538315776 bytes
2024-05-17T15:10:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2628076544 bytes
2024-05-17T15:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2670853120 bytes
2024-05-17T15:10:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2695519232 bytes
2024-05-17T15:10:09.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2718841856 bytes
2024-05-17T15:10:09.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2719637504 bytes
2024-05-17T15:10:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2760116224 bytes
2024-05-17T15:10:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2800036864 bytes
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_13
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_13 128M
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:10:10.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:10:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:10:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:10:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_13
2024-05-17T15:10:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_13 snapshot
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:10.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot
2024-05-17T15:10:11.436 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:10:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_13
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_13':
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13057fd65e62
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13057fd65e62
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:10:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:10 2024
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:10 2024
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:10 2024
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 2c4982c8-61ec-4697-b3aa-46a9ce03b746
2024-05-17T15:10:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_13
2024-05-17T15:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:10:11.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:10:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:10:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:10:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:10:11.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 74345472 bytes
2024-05-17T15:10:11.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 89702400 bytes
2024-05-17T15:10:12.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 113255424 bytes
2024-05-17T15:10:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 144337920 bytes
2024-05-17T15:10:12.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 247156736 bytes
2024-05-17T15:10:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 294693888 bytes
2024-05-17T15:10:12.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 338498560 bytes
2024-05-17T15:10:13.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 383027200 bytes
2024-05-17T15:10:13.358 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 423374848 bytes
2024-05-17T15:10:13.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 506978304 bytes
2024-05-17T15:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 529897472 bytes
2024-05-17T15:10:13.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 536365056 bytes
2024-05-17T15:10:14.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 600974336 bytes
2024-05-17T15:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 642647040 bytes
2024-05-17T15:10:14.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 686668800 bytes
2024-05-17T15:10:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 701644800 bytes
2024-05-17T15:10:14.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 724929536 bytes
2024-05-17T15:10:14.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 741542912 bytes
2024-05-17T15:10:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 819407872 bytes
2024-05-17T15:10:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 855192576 bytes
2024-05-17T15:10:15.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 908690432 bytes
2024-05-17T15:10:15.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 919497728 bytes
2024-05-17T15:10:15.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 945607680 bytes
2024-05-17T15:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1049738240 bytes
2024-05-17T15:10:16.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1060660224 bytes
2024-05-17T15:10:16.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1091951616 bytes
2024-05-17T15:10:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1153560576 bytes
2024-05-17T15:10:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1168956416 bytes
2024-05-17T15:10:16.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1185693696 bytes
2024-05-17T15:10:17.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1207863296 bytes
2024-05-17T15:10:17.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1273422848 bytes
2024-05-17T15:10:17.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1341632512 bytes
2024-05-17T15:10:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1375768576 bytes
2024-05-17T15:10:17.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1397456896 bytes
2024-05-17T15:10:17.951 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1434143744 bytes
2024-05-17T15:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1462378496 bytes
2024-05-17T15:10:18.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1586805760 bytes
2024-05-17T15:10:18.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1678438400 bytes
2024-05-17T15:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1723085824 bytes
2024-05-17T15:10:19.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1755143168 bytes
2024-05-17T15:10:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1826925568 bytes
2024-05-17T15:10:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1857917952 bytes
2024-05-17T15:10:19.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1899556864 bytes
2024-05-17T15:10:19.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1950186496 bytes
2024-05-17T15:10:19.992 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1970968576 bytes
2024-05-17T15:10:20.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2020292608 bytes
2024-05-17T15:10:20.340 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2030331904 bytes
2024-05-17T15:10:20.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2116092928 bytes
2024-05-17T15:10:20.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2198256640 bytes
2024-05-17T15:10:21.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2261811200 bytes
2024-05-17T15:10:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2292915200 bytes
2024-05-17T15:10:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2295626752 bytes
2024-05-17T15:10:21.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2314294272 bytes
2024-05-17T15:10:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2419435520 bytes
2024-05-17T15:10:21.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2468435968 bytes
2024-05-17T15:10:22.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2543469568 bytes
2024-05-17T15:10:22.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2586878976 bytes
2024-05-17T15:10:22.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2621786112 bytes
2024-05-17T15:10:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2694511616 bytes
2024-05-17T15:10:23.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.520604825055669) > weight (0.25)
2024-05-17T15:10:23.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4153522128987629) > weight (0.25)
2024-05-17T15:10:23.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7606989980651128) > weight (0.25)
2024-05-17T15:10:23.022 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9302058311207139) > weight (0.25)
2024-05-17T15:10:23.022 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing
2024-05-17T15:10:23.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2772744192 bytes
2024-05-17T15:10:23.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2811143168 bytes
2024-05-17T15:10:23.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2869081088 bytes
2024-05-17T15:10:23.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2955121664 bytes
2024-05-17T15:10:23.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2995895296 bytes
2024-05-17T15:10:24.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 3011582976 bytes
2024-05-17T15:10:24.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 3117616128 bytes
2024-05-17T15:10:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 3166963712 bytes
2024-05-17T15:10:24.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 3179603968 bytes
2024-05-17T15:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 3214219264 bytes
2024-05-17T15:10:25.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 3250715648 bytes
2024-05-17T15:10:25.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3346219008 bytes
2024-05-17T15:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_14
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_14 128M
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_14
2024-05-17T15:10:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_14 snapshot
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot
2024-05-17T15:10:26.029 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0
2024-05-17T15:10:26.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15
2024-05-17T15:10:26.030 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons
2024-05-17T15:10:26.031 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T15:10:26.030+0000 7f83708f3640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 33397) UID: 1000
2024-05-17T15:10:26.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:10:26.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_14
2024-05-17T15:10:26.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_14':
2024-05-17T15:10:26.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:10:26.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13118cf1af03
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13118cf1af03
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:25 2024
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:25 2024
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:25 2024
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:10:26.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 0b9e4c5c-db1d-41ce-a537-9dcae0dfcc10
2024-05-17T15:10:26.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:10:26.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:10:26.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_14
2024-05-17T15:10:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:10:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:10:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=8
2024-05-17T15:10:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:10:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:10:26.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 19611648 bytes
2024-05-17T15:10:26.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 40413184 bytes
2024-05-17T15:10:27.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 92464128 bytes
2024-05-17T15:10:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 127364096 bytes
2024-05-17T15:10:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 172662784 bytes
2024-05-17T15:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 213925888 bytes
2024-05-17T15:10:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 287285248 bytes
2024-05-17T15:10:28.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 324685824 bytes
2024-05-17T15:10:28.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 381441024 bytes
2024-05-17T15:10:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 420191232 bytes
2024-05-17T15:10:28.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 478415872 bytes
2024-05-17T15:10:28.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 499177472 bytes
2024-05-17T15:10:29.031 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0
2024-05-17T15:10:29.031 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:10:29.032 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0
2024-05-17T15:10:29.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon
2024-05-17T15:10:29.032 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0
2024-05-17T15:10:29.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 541026304 bytes
2024-05-17T15:10:29.035 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started
2024-05-17T15:10:29.035 INFO:tasks.rbd_mirror_thrash:waiting for 116 secs before thrashing
2024-05-17T15:10:29.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 546560000 bytes
2024-05-17T15:10:29.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 611385344 bytes
2024-05-17T15:10:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 653314048 bytes
2024-05-17T15:10:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 702314496 bytes
2024-05-17T15:10:29.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 756599808 bytes
2024-05-17T15:10:30.134 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 791057408 bytes
2024-05-17T15:10:30.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 876106752 bytes
2024-05-17T15:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 887627776 bytes
2024-05-17T15:10:30.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 909460480 bytes
2024-05-17T15:10:30.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 966109184 bytes
2024-05-17T15:10:31.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1003824128 bytes
2024-05-17T15:10:31.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1021889536 bytes
2024-05-17T15:10:31.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1059439616 bytes
2024-05-17T15:10:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1087017984 bytes
2024-05-17T15:10:31.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1154615296 bytes
2024-05-17T15:10:31.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1189093376 bytes
2024-05-17T15:10:32.155 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1223163904 bytes
2024-05-17T15:10:32.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1324634112 bytes
2024-05-17T15:10:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1351704576 bytes
2024-05-17T15:10:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1375808512 bytes
2024-05-17T15:10:32.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1386295296 bytes
2024-05-17T15:10:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1422490624 bytes
2024-05-17T15:10:33.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1470014464 bytes
2024-05-17T15:10:33.451 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1500043264 bytes
2024-05-17T15:10:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1594890240 bytes
2024-05-17T15:10:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1637286912 bytes
2024-05-17T15:10:34.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1678171136 bytes
2024-05-17T15:10:34.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1737350144 bytes
2024-05-17T15:10:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_15
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:10:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_15 128M
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:34.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_15
2024-05-17T15:10:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:10:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:10:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_15 snapshot
2024-05-17T15:10:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:10:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot
2024-05-17T15:10:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:10:35.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_15
2024-05-17T15:10:35.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_15':
2024-05-17T15:10:35.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1321e1b02811
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1321e1b02811
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:10:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:34 2024
2024-05-17T15:10:35.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:34 2024
2024-05-17T15:10:35.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:34 2024
2024-05-17T15:10:35.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:10:35.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:10:35.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 4963d81f-f6e8-40ba-b80c-6c7d9bd6f465
2024-05-17T15:10:35.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:10:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:10:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_15
2024-05-17T15:10:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:35.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:10:35.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:10:35.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=6
2024-05-17T15:10:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:10:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:10:35.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 57269248 bytes
2024-05-17T15:10:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 57906176 bytes
2024-05-17T15:10:36.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 137017344 bytes
2024-05-17T15:10:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 141236224 bytes
2024-05-17T15:10:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 156005376 bytes
2024-05-17T15:10:36.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 232845312 bytes
2024-05-17T15:10:36.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 265456640 bytes
2024-05-17T15:10:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 308036608 bytes
2024-05-17T15:10:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 322112512 bytes
2024-05-17T15:10:37.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 397288448 bytes
2024-05-17T15:10:37.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 419017728 bytes
2024-05-17T15:10:37.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 454389760 bytes
2024-05-17T15:10:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 518060032 bytes
2024-05-17T15:10:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 534708224 bytes
2024-05-17T15:10:38.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 557920256 bytes
2024-05-17T15:10:38.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 575281152 bytes
2024-05-17T15:10:38.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 642309120 bytes
2024-05-17T15:10:38.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 680088576 bytes
2024-05-17T15:10:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 713133056 bytes
2024-05-17T15:10:39.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 713743360 bytes
2024-05-17T15:10:39.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 726831104 bytes
2024-05-17T15:10:39.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 797827072 bytes
2024-05-17T15:10:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 832567296 bytes
2024-05-17T15:10:39.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 889121792 bytes
2024-05-17T15:10:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1004327936 bytes
2024-05-17T15:10:40.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1056080896 bytes
2024-05-17T15:10:40.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1131635712 bytes
2024-05-17T15:10:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1256869888 bytes
2024-05-17T15:10:41.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1261747200 bytes
2024-05-17T15:10:41.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1300192256 bytes
2024-05-17T15:10:41.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1338541056 bytes
2024-05-17T15:10:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1395004416 bytes
2024-05-17T15:10:41.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_16
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_16 128M
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_16
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_16 snapshot
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:10:41.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:41.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot
2024-05-17T15:10:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:10:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_16
2024-05-17T15:10:42.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_16':
2024-05-17T15:10:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:10:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:10:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:10:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 132dbf9399e4
2024-05-17T15:10:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.132dbf9399e4
2024-05-17T15:10:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:10:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:10:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:10:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:10:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:41 2024
2024-05-17T15:10:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:41 2024
2024-05-17T15:10:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:41 2024
2024-05-17T15:10:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:10:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:10:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: e802db82-1a1d-42f1-9c16-9e915554bd00
2024-05-17T15:10:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:10:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:10:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_16
2024-05-17T15:10:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:42.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:42.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:10:42.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:10:42.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=10
2024-05-17T15:10:42.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:10:42.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:10:42.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 100832256 bytes
2024-05-17T15:10:43.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 132318208 bytes
2024-05-17T15:10:43.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 210044928 bytes
2024-05-17T15:10:43.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 293120000 bytes
2024-05-17T15:10:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 318749696 bytes
2024-05-17T15:10:44.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 364888064 bytes
2024-05-17T15:10:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 373984256 bytes
2024-05-17T15:10:44.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 404462592 bytes
2024-05-17T15:10:44.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 410040320 bytes
2024-05-17T15:10:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 437090304 bytes
2024-05-17T15:10:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 474663936 bytes
2024-05-17T15:10:45.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 538235904 bytes
2024-05-17T15:10:45.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 580754432 bytes
2024-05-17T15:10:45.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 613361664 bytes
2024-05-17T15:10:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 653813760 bytes
2024-05-17T15:10:45.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 721859584 bytes
2024-05-17T15:10:46.083 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 791535616 bytes
2024-05-17T15:10:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 810813440 bytes
2024-05-17T15:10:46.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 887971840 bytes
2024-05-17T15:10:46.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 927378432 bytes
2024-05-17T15:10:46.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 928000000 bytes
2024-05-17T15:10:47.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 999688192 bytes
2024-05-17T15:10:47.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1045426176 bytes
2024-05-17T15:10:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1130705920 bytes
2024-05-17T15:10:47.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1167675392 bytes
2024-05-17T15:10:47.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1189612544 bytes
2024-05-17T15:10:48.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1269842944 bytes
2024-05-17T15:10:48.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1295352832 bytes
2024-05-17T15:10:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1299525632 bytes
2024-05-17T15:10:48.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1366917120 bytes
2024-05-17T15:10:48.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1408039936 bytes
2024-05-17T15:10:49.036 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1477339136 bytes
2024-05-17T15:10:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1552216064 bytes
2024-05-17T15:10:49.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1606943744 bytes
2024-05-17T15:10:49.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1719391232 bytes
2024-05-17T15:10:50.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1798133760 bytes
2024-05-17T15:10:50.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1812399104 bytes
2024-05-17T15:10:50.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1859333120 bytes
2024-05-17T15:10:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1903366144 bytes
2024-05-17T15:10:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1934603264 bytes
2024-05-17T15:10:50.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1996939264 bytes
2024-05-17T15:10:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 2086353920 bytes
2024-05-17T15:10:51.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 2086978560 bytes
2024-05-17T15:10:51.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 2121759744 bytes
2024-05-17T15:10:51.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 2127775744 bytes
2024-05-17T15:10:51.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2172517376 bytes
2024-05-17T15:10:52.152 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2274755584 bytes
2024-05-17T15:10:52.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2334984192 bytes
2024-05-17T15:10:52.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2390598656 bytes
2024-05-17T15:10:52.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:10:52.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:10:52.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_17
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_17 128M
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:10:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:10:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:10:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:10:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:10:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_17
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_17 snapshot
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:10:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot
2024-05-17T15:10:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:10:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_17
2024-05-17T15:10:53.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_17':
2024-05-17T15:10:53.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:10:53.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1339221a2ec6
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1339221a2ec6
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:52 2024
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:52 2024
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:52 2024
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 51b71ae3-ecdd-405e-b221-6cbc4ee1b11e
2024-05-17T15:10:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:10:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:10:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_17
2024-05-17T15:10:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:10:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:10:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:10:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:10:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:10:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:10:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:10:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 46252032 bytes
2024-05-17T15:10:54.083 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 56434688 bytes
2024-05-17T15:10:54.350 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 139801600 bytes
2024-05-17T15:10:54.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 169295872 bytes
2024-05-17T15:10:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 207517696 bytes
2024-05-17T15:10:54.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 219887616 bytes
2024-05-17T15:10:55.097 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 268339200 bytes
2024-05-17T15:10:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 319126528 bytes
2024-05-17T15:10:55.469 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 349651968 bytes
2024-05-17T15:10:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 414078976 bytes
2024-05-17T15:10:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 424945664 bytes
2024-05-17T15:10:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 436451328 bytes
2024-05-17T15:10:56.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 474799104 bytes
2024-05-17T15:10:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 501575680 bytes
2024-05-17T15:10:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 511176704 bytes
2024-05-17T15:10:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 530881536 bytes
2024-05-17T15:10:56.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 563666944 bytes
2024-05-17T15:10:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 617407488 bytes
2024-05-17T15:10:57.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 656053248 bytes
2024-05-17T15:10:57.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 680029184 bytes
2024-05-17T15:10:57.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 721333248 bytes
2024-05-17T15:10:57.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 749575168 bytes
2024-05-17T15:10:57.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 751572992 bytes
2024-05-17T15:10:57.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 800249856 bytes
2024-05-17T15:10:58.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 820406272 bytes
2024-05-17T15:10:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 854540288 bytes
2024-05-17T15:10:58.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 889006080 bytes
2024-05-17T15:10:58.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 998586368 bytes
2024-05-17T15:10:58.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1046477824 bytes
2024-05-17T15:10:59.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1125501952 bytes
2024-05-17T15:10:59.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1197709312 bytes
2024-05-17T15:10:59.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1244812288 bytes
2024-05-17T15:10:59.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1296606208 bytes
2024-05-17T15:11:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1329786880 bytes
2024-05-17T15:11:00.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1375166464 bytes
2024-05-17T15:11:00.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1421094912 bytes
2024-05-17T15:11:00.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1465462784 bytes
2024-05-17T15:11:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1531255808 bytes
2024-05-17T15:11:01.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1608464384 bytes
2024-05-17T15:11:01.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1610643456 bytes
2024-05-17T15:11:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1621480448 bytes
2024-05-17T15:11:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1661204480 bytes
2024-05-17T15:11:01.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1682973696 bytes
2024-05-17T15:11:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1730082816 bytes
2024-05-17T15:11:02.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1753661440 bytes
2024-05-17T15:11:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1794470912 bytes
2024-05-17T15:11:02.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1833533440 bytes
2024-05-17T15:11:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1896632320 bytes
2024-05-17T15:11:02.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1938976768 bytes
2024-05-17T15:11:03.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 1972495360 bytes
2024-05-17T15:11:03.250 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2000629760 bytes
2024-05-17T15:11:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2012900352 bytes
2024-05-17T15:11:03.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2038863872 bytes
2024-05-17T15:11:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2109087744 bytes
2024-05-17T15:11:03.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2126612480 bytes
2024-05-17T15:11:04.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2128083968 bytes
2024-05-17T15:11:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2164236288 bytes
2024-05-17T15:11:04.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2207702016 bytes
2024-05-17T15:11:04.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2253443072 bytes
2024-05-17T15:11:04.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2277276672 bytes
2024-05-17T15:11:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2334386176 bytes
2024-05-17T15:11:05.331 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2453378048 bytes
2024-05-17T15:11:05.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2469243904 bytes
2024-05-17T15:11:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2531256320 bytes
2024-05-17T15:11:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2532403200 bytes
2024-05-17T15:11:06.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2614226944 bytes
2024-05-17T15:11:06.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2647400448 bytes
2024-05-17T15:11:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2660037632 bytes
2024-05-17T15:11:06.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2687327232 bytes
2024-05-17T15:11:06.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2717360128 bytes
2024-05-17T15:11:06.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:11:06.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:11:06.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:11:06.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:06.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_18
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_18 128M
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_18
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_18 snapshot
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:11:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot
2024-05-17T15:11:07.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:11:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_18
2024-05-17T15:11:07.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_18':
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13453f9a18c3
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13453f9a18c3
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:11:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:06 2024
2024-05-17T15:11:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:06 2024
2024-05-17T15:11:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:06 2024
2024-05-17T15:11:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:11:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:11:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: bc8b79fe-c60a-42f2-8eff-db2c0775304b
2024-05-17T15:11:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:11:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:11:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_18
2024-05-17T15:11:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:11:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:11:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 33878016 bytes
2024-05-17T15:11:08.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 77109248 bytes
2024-05-17T15:11:08.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 117772288 bytes
2024-05-17T15:11:08.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 173508608 bytes
2024-05-17T15:11:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 188122112 bytes
2024-05-17T15:11:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 253252608 bytes
2024-05-17T15:11:09.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 324299776 bytes
2024-05-17T15:11:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 352615424 bytes
2024-05-17T15:11:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 385103872 bytes
2024-05-17T15:11:09.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 426277888 bytes
2024-05-17T15:11:09.828 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:11:09.825+0000 7f905860c640 -1 reset not still connected to 0x55d26deb56c0
2024-05-17T15:11:09.828 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:11:09.825+0000 7f905860c640 -1 reset not still connected to 0x55d279fd1c70
2024-05-17T15:11:09.828 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:11:09.825+0000 7f905860c640 -1 reset not still connected to 0x55d27eb4f040
2024-05-17T15:11:09.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 463058944 bytes
2024-05-17T15:11:10.083 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 506868736 bytes
2024-05-17T15:11:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 543488000 bytes
2024-05-17T15:11:10.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 562323456 bytes
2024-05-17T15:11:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 616658944 bytes
2024-05-17T15:11:10.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 695013376 bytes
2024-05-17T15:11:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 695628800 bytes
2024-05-17T15:11:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 738761728 bytes
2024-05-17T15:11:11.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 853553152 bytes
2024-05-17T15:11:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 876293120 bytes
2024-05-17T15:11:11.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 965202944 bytes
2024-05-17T15:11:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1003782144 bytes
2024-05-17T15:11:12.204 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1004398592 bytes
2024-05-17T15:11:12.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1062803456 bytes
2024-05-17T15:11:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1115257856 bytes
2024-05-17T15:11:12.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1157395456 bytes
2024-05-17T15:11:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1183673344 bytes
2024-05-17T15:11:13.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1209888768 bytes
2024-05-17T15:11:13.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1274592256 bytes
2024-05-17T15:11:13.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1322546176 bytes
2024-05-17T15:11:13.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1343191040 bytes
2024-05-17T15:11:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1374281728 bytes
2024-05-17T15:11:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1413249024 bytes
2024-05-17T15:11:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1428814848 bytes
2024-05-17T15:11:14.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1465084928 bytes
2024-05-17T15:11:14.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1516002304 bytes
2024-05-17T15:11:14.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1575639040 bytes
2024-05-17T15:11:14.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1593858048 bytes
2024-05-17T15:11:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1596761088 bytes
2024-05-17T15:11:15.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1631819776 bytes
2024-05-17T15:11:15.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1706668032 bytes
2024-05-17T15:11:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1788907520 bytes
2024-05-17T15:11:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1840339968 bytes
2024-05-17T15:11:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1912941568 bytes
2024-05-17T15:11:16.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1998045184 bytes
2024-05-17T15:11:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2008255488 bytes
2024-05-17T15:11:16.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2052257792 bytes
2024-05-17T15:11:16.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2101131264 bytes
2024-05-17T15:11:17.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2133549056 bytes
2024-05-17T15:11:17.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2184690688 bytes
2024-05-17T15:11:17.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2249237504 bytes
2024-05-17T15:11:17.701 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T15:11:17.701+0000 7fc89825a640 -1 reset not still connected to 0x5588f11825b0
2024-05-17T15:11:17.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2279117824 bytes
2024-05-17T15:11:17.937 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2323865600 bytes
2024-05-17T15:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2356785152 bytes
2024-05-17T15:11:18.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2379615232 bytes
2024-05-17T15:11:18.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2410563584 bytes
2024-05-17T15:11:18.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2460326912 bytes
2024-05-17T15:11:18.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2490375168 bytes
2024-05-17T15:11:18.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2515182592 bytes
2024-05-17T15:11:19.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2569847808 bytes
2024-05-17T15:11:19.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2638943232 bytes
2024-05-17T15:11:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2639562752 bytes
2024-05-17T15:11:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2697781248 bytes
2024-05-17T15:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2733058048 bytes
2024-05-17T15:11:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2750662656 bytes
2024-05-17T15:11:20.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2789338112 bytes
2024-05-17T15:11:20.406 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2813858816 bytes
2024-05-17T15:11:20.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2910166016 bytes
2024-05-17T15:11:20.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2927627264 bytes
2024-05-17T15:11:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2960110592 bytes
2024-05-17T15:11:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3001796608 bytes
2024-05-17T15:11:21.279 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:11:21.277+0000 7f28b79e2640 -1 reset not still connected to 0x5625bdf5e8f0
2024-05-17T15:11:21.418 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3050075136 bytes
2024-05-17T15:11:21.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 3056039936 bytes
2024-05-17T15:11:21.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 3081643008 bytes
2024-05-17T15:11:21.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:11:21.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:11:21.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:11:21.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:21.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_19
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_19 128M
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_19
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_19 snapshot
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:21.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:11:21.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:21.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot
2024-05-17T15:11:22.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:11:22.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_19
2024-05-17T15:11:22.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_19':
2024-05-17T15:11:22.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:11:22.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:11:22.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:11:22.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1351fb554acb
2024-05-17T15:11:22.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1351fb554acb
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:21 2024
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:21 2024
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:21 2024
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 0c1ecc0c-82a3-4c9c-96a6-bc3f7117e70a
2024-05-17T15:11:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:11:22.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:11:22.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_19
2024-05-17T15:11:22.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:22.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:22.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:11:22.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:11:22.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=8
2024-05-17T15:11:22.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:11:22.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:11:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 49027072 bytes
2024-05-17T15:11:22.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 109330432 bytes
2024-05-17T15:11:23.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 155966464 bytes
2024-05-17T15:11:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 188270592 bytes
2024-05-17T15:11:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 212383744 bytes
2024-05-17T15:11:23.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 270149632 bytes
2024-05-17T15:11:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 293324800 bytes
2024-05-17T15:11:23.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 314190848 bytes
2024-05-17T15:11:24.023 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0
2024-05-17T15:11:24.024 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15
2024-05-17T15:11:24.024 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons
2024-05-17T15:11:24.024 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T15:11:24.025+0000 7f9cb7871640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0  (PID: 31508) UID: 1000
2024-05-17T15:11:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 320527360 bytes
2024-05-17T15:11:24.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 378369024 bytes
2024-05-17T15:11:24.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 420692992 bytes
2024-05-17T15:11:24.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 439603200 bytes
2024-05-17T15:11:24.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 464498688 bytes
2024-05-17T15:11:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 505790464 bytes
2024-05-17T15:11:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 565100544 bytes
2024-05-17T15:11:25.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 603540480 bytes
2024-05-17T15:11:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 668411904 bytes
2024-05-17T15:11:25.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 696847360 bytes
2024-05-17T15:11:25.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 761505792 bytes
2024-05-17T15:11:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 804804608 bytes
2024-05-17T15:11:26.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 914520064 bytes
2024-05-17T15:11:26.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 941741056 bytes
2024-05-17T15:11:26.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 942367744 bytes
2024-05-17T15:11:26.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 962550784 bytes
2024-05-17T15:11:27.025 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0
2024-05-17T15:11:27.025 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:11:27.025 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0
2024-05-17T15:11:27.026 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon
2024-05-17T15:11:27.026 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0
2024-05-17T15:11:27.028 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started
2024-05-17T15:11:27.029 INFO:tasks.rbd_mirror_thrash:waiting for 76 secs before thrashing
2024-05-17T15:11:27.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 993205248 bytes
2024-05-17T15:11:27.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1073243136 bytes
2024-05-17T15:11:27.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1116516352 bytes
2024-05-17T15:11:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1152773120 bytes
2024-05-17T15:11:27.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1210194944 bytes
2024-05-17T15:11:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1243167744 bytes
2024-05-17T15:11:28.284 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1273031680 bytes
2024-05-17T15:11:28.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1335456768 bytes
2024-05-17T15:11:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1378373632 bytes
2024-05-17T15:11:28.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1378971648 bytes
2024-05-17T15:11:28.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1425862656 bytes
2024-05-17T15:11:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1536116736 bytes
2024-05-17T15:11:29.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1627816960 bytes
2024-05-17T15:11:29.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1679069184 bytes
2024-05-17T15:11:30.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1723871232 bytes
2024-05-17T15:11:30.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1814930432 bytes
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_20
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_20 128M
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:11:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:11:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:11:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:11:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_20
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_20 snapshot
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:11:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot
2024-05-17T15:11:31.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:11:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_20
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_20':
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13674a9f153b
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13674a9f153b
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:11:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:30 2024
2024-05-17T15:11:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:30 2024
2024-05-17T15:11:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:30 2024
2024-05-17T15:11:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:11:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:11:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 1f368bc9-6917-4e4c-a45d-3582c17ee0f7
2024-05-17T15:11:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:11:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:11:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_20
2024-05-17T15:11:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:11:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:11:31.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:11:31.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:11:31.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:11:31.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 41825280 bytes
2024-05-17T15:11:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 70874112 bytes
2024-05-17T15:11:32.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 128984064 bytes
2024-05-17T15:11:32.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 209846272 bytes
2024-05-17T15:11:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 265222144 bytes
2024-05-17T15:11:32.753 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 265799680 bytes
2024-05-17T15:11:32.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 278562816 bytes
2024-05-17T15:11:33.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 343109632 bytes
2024-05-17T15:11:33.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 384149504 bytes
2024-05-17T15:11:33.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 407863296 bytes
2024-05-17T15:11:33.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 418244608 bytes
2024-05-17T15:11:33.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 463750144 bytes
2024-05-17T15:11:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 510961664 bytes
2024-05-17T15:11:34.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 618543104 bytes
2024-05-17T15:11:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 661681152 bytes
2024-05-17T15:11:34.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 705549312 bytes
2024-05-17T15:11:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 713905152 bytes
2024-05-17T15:11:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 714527744 bytes
2024-05-17T15:11:35.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 750990336 bytes
2024-05-17T15:11:35.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 751605760 bytes
2024-05-17T15:11:35.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 773431296 bytes
2024-05-17T15:11:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 803699712 bytes
2024-05-17T15:11:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 833634304 bytes
2024-05-17T15:11:35.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 873832448 bytes
2024-05-17T15:11:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 899016704 bytes
2024-05-17T15:11:36.370 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 989435904 bytes
2024-05-17T15:11:36.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1047330816 bytes
2024-05-17T15:11:36.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1067453440 bytes
2024-05-17T15:11:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1117125632 bytes
2024-05-17T15:11:37.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1156465664 bytes
2024-05-17T15:11:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1200306176 bytes
2024-05-17T15:11:37.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1227141120 bytes
2024-05-17T15:11:37.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1282977792 bytes
2024-05-17T15:11:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1332922368 bytes
2024-05-17T15:11:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1406827520 bytes
2024-05-17T15:11:38.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1407453184 bytes
2024-05-17T15:11:38.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1426942976 bytes
2024-05-17T15:11:38.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1493640192 bytes
2024-05-17T15:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1543646208 bytes
2024-05-17T15:11:38.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1588800512 bytes
2024-05-17T15:11:39.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1624792064 bytes
2024-05-17T15:11:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1625729024 bytes
2024-05-17T15:11:39.372 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1629480960 bytes
2024-05-17T15:11:39.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1684631552 bytes
2024-05-17T15:11:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1719444480 bytes
2024-05-17T15:11:39.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1726812160 bytes
2024-05-17T15:11:40.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1763828736 bytes
2024-05-17T15:11:40.230 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1786035200 bytes
2024-05-17T15:11:40.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1827957760 bytes
2024-05-17T15:11:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 1868406784 bytes
2024-05-17T15:11:40.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 1874088960 bytes
2024-05-17T15:11:40.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 1897781248 bytes
2024-05-17T15:11:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 1944138752 bytes
2024-05-17T15:11:41.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2077993984 bytes
2024-05-17T15:11:41.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2096072704 bytes
2024-05-17T15:11:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2184387584 bytes
2024-05-17T15:11:42.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2231615488 bytes
2024-05-17T15:11:42.288 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2301963264 bytes
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_21
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_21 128M
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:11:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:11:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:11:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_21
2024-05-17T15:11:42.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:11:42.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:42.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:11:42.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:42.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_21 snapshot
2024-05-17T15:11:42.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:42.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:42.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:11:42.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:42.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot
2024-05-17T15:11:43.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:11:43.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_21
2024-05-17T15:11:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_21':
2024-05-17T15:11:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:11:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:11:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1373e856ec73
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1373e856ec73
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:42 2024
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:42 2024
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:42 2024
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7d839829-ed18-4ca7-9b9a-0db47f33f6f5
2024-05-17T15:11:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:11:43.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:11:43.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_21
2024-05-17T15:11:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:11:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:11:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=5
2024-05-17T15:11:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:11:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:11:43.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 37610496 bytes
2024-05-17T15:11:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 92813312 bytes
2024-05-17T15:11:44.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 144686080 bytes
2024-05-17T15:11:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 172735488 bytes
2024-05-17T15:11:44.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 173320192 bytes
2024-05-17T15:11:44.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 215861248 bytes
2024-05-17T15:11:44.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 270052352 bytes
2024-05-17T15:11:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 313713664 bytes
2024-05-17T15:11:45.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 335992832 bytes
2024-05-17T15:11:45.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 354377728 bytes
2024-05-17T15:11:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 382220288 bytes
2024-05-17T15:11:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 417601536 bytes
2024-05-17T15:11:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 450586624 bytes
2024-05-17T15:11:46.031 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T15:11:46.032+0000 7fc89825a640 -1 reset not still connected to 0x5588f0aa5e10
2024-05-17T15:11:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 509212672 bytes
2024-05-17T15:11:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 523698176 bytes
2024-05-17T15:11:46.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 543276032 bytes
2024-05-17T15:11:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 622902272 bytes
2024-05-17T15:11:46.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 670217216 bytes
2024-05-17T15:11:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 738960384 bytes
2024-05-17T15:11:47.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 786556928 bytes
2024-05-17T15:11:47.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 826705920 bytes
2024-05-17T15:11:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 864858112 bytes
2024-05-17T15:11:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 892696576 bytes
2024-05-17T15:11:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 914234368 bytes
2024-05-17T15:11:48.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 926914560 bytes
2024-05-17T15:11:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 932524032 bytes
2024-05-17T15:11:48.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 979364864 bytes
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_22
2024-05-17T15:11:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_22 128M
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:11:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:11:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:11:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:11:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:11:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_22
2024-05-17T15:11:48.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:11:48.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:48.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:11:48.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:11:48.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_22 snapshot
2024-05-17T15:11:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:11:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:11:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot
2024-05-17T15:11:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:11:49.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_22
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_22':
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 137fd57887a6
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.137fd57887a6
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:11:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:48 2024
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:48 2024
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:48 2024
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: e1e9a751-7f3a-4c40-a3f6-7a2a2551ce54
2024-05-17T15:11:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:11:49.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:11:49.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_22
2024-05-17T15:11:49.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:11:49.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:11:49.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:11:49.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:11:49.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:11:49.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:11:49.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:11:49.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 95954944 bytes
2024-05-17T15:11:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 190172160 bytes
2024-05-17T15:11:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 241163264 bytes
2024-05-17T15:11:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 273971200 bytes
2024-05-17T15:11:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 308758528 bytes
2024-05-17T15:11:51.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 407719936 bytes
2024-05-17T15:11:51.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 427365376 bytes
2024-05-17T15:11:51.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 441750528 bytes
2024-05-17T15:11:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 457656320 bytes
2024-05-17T15:11:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 528080896 bytes
2024-05-17T15:11:51.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 609568768 bytes
2024-05-17T15:11:52.083 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 611469312 bytes
2024-05-17T15:11:52.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 640128000 bytes
2024-05-17T15:11:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 701990912 bytes
2024-05-17T15:11:52.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 811595776 bytes
2024-05-17T15:11:53.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 877343744 bytes
2024-05-17T15:11:53.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 910229504 bytes
2024-05-17T15:11:53.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 1016674304 bytes
2024-05-17T15:11:53.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 1078321152 bytes
2024-05-17T15:11:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 1105470464 bytes
2024-05-17T15:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1199794176 bytes
2024-05-17T15:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1257326592 bytes
2024-05-17T15:11:54.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1276173312 bytes
2024-05-17T15:11:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1305402368 bytes
2024-05-17T15:11:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1333045248 bytes
2024-05-17T15:11:55.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1391717376 bytes
2024-05-17T15:11:55.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1417067520 bytes
2024-05-17T15:11:55.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1462132736 bytes
2024-05-17T15:11:55.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1495058432 bytes
2024-05-17T15:11:55.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1526350848 bytes
2024-05-17T15:11:56.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1595032576 bytes
2024-05-17T15:11:56.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1619318784 bytes
2024-05-17T15:11:56.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1646819328 bytes
2024-05-17T15:11:56.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1700088832 bytes
2024-05-17T15:11:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1768686592 bytes
2024-05-17T15:11:56.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1793343488 bytes
2024-05-17T15:11:57.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1845929984 bytes
2024-05-17T15:11:57.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1868578816 bytes
2024-05-17T15:11:57.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1906864128 bytes
2024-05-17T15:11:57.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1907885056 bytes
2024-05-17T15:11:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1954174976 bytes
2024-05-17T15:11:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1974976512 bytes
2024-05-17T15:11:58.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 2010752000 bytes
2024-05-17T15:11:58.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 2022994944 bytes
2024-05-17T15:11:58.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 2054952960 bytes
2024-05-17T15:11:58.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2108549120 bytes
2024-05-17T15:11:58.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2151947264 bytes
2024-05-17T15:11:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2178341888 bytes
2024-05-17T15:11:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2192017408 bytes
2024-05-17T15:11:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2228754432 bytes
2024-05-17T15:11:59.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2280647680 bytes
2024-05-17T15:11:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2317675520 bytes
2024-05-17T15:11:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2325511168 bytes
2024-05-17T15:11:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2343188480 bytes
2024-05-17T15:12:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2392373248 bytes
2024-05-17T15:12:00.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2411948032 bytes
2024-05-17T15:12:00.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2447879168 bytes
2024-05-17T15:12:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_23
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_23 128M
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_23
2024-05-17T15:12:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_23 snapshot
2024-05-17T15:12:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:12:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot
2024-05-17T15:12:01.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_23
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_23':
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 138bf9305553
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.138bf9305553
2024-05-17T15:12:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:00 2024
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:00 2024
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:00 2024
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 21cc1395-0501-4467-8166-6ae00529e254
2024-05-17T15:12:01.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_23
2024-05-17T15:12:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:12:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:12:01.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:01.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 66251776 bytes
2024-05-17T15:12:02.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 126913536 bytes
2024-05-17T15:12:02.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 137962496 bytes
2024-05-17T15:12:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 190224384 bytes
2024-05-17T15:12:02.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 208795648 bytes
2024-05-17T15:12:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 229392384 bytes
2024-05-17T15:12:03.215 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 269122560 bytes
2024-05-17T15:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 333429760 bytes
2024-05-17T15:12:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 378585088 bytes
2024-05-17T15:12:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 400083968 bytes
2024-05-17T15:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 466963456 bytes
2024-05-17T15:12:04.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 502536192 bytes
2024-05-17T15:12:04.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 566810624 bytes
2024-05-17T15:12:04.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 605362176 bytes
2024-05-17T15:12:04.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 633589760 bytes
2024-05-17T15:12:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 656937984 bytes
2024-05-17T15:12:05.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 678909952 bytes
2024-05-17T15:12:05.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 696381440 bytes
2024-05-17T15:12:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 757235712 bytes
2024-05-17T15:12:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 815730688 bytes
2024-05-17T15:12:05.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 860733440 bytes
2024-05-17T15:12:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 906649600 bytes
2024-05-17T15:12:06.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 941597696 bytes
2024-05-17T15:12:06.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 997422080 bytes
2024-05-17T15:12:06.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1046410240 bytes
2024-05-17T15:12:06.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1090356224 bytes
2024-05-17T15:12:06.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1111542784 bytes
2024-05-17T15:12:07.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1130175488 bytes
2024-05-17T15:12:07.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1174792192 bytes
2024-05-17T15:12:07.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1215501312 bytes
2024-05-17T15:12:07.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1272695808 bytes
2024-05-17T15:12:07.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1303074816 bytes
2024-05-17T15:12:08.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1358107648 bytes
2024-05-17T15:12:08.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1420100608 bytes
2024-05-17T15:12:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1444213760 bytes
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_24
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:08.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_24 128M
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_24
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_24 snapshot
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot
2024-05-17T15:12:09.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_24
2024-05-17T15:12:09.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_24':
2024-05-17T15:12:09.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:09.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13971afb53a9
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13971afb53a9
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:08 2024
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:08 2024
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:08 2024
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7f23dbe5-352e-45f0-ba06-9261e41a7df4
2024-05-17T15:12:09.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:09.683 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:09.683 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_24
2024-05-17T15:12:09.683 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:12:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=6
2024-05-17T15:12:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:09.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 30964736 bytes
2024-05-17T15:12:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 31584256 bytes
2024-05-17T15:12:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 77971456 bytes
2024-05-17T15:12:10.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 149788672 bytes
2024-05-17T15:12:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 196962304 bytes
2024-05-17T15:12:10.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 228832256 bytes
2024-05-17T15:12:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 246203392 bytes
2024-05-17T15:12:11.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 279565312 bytes
2024-05-17T15:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 332220416 bytes
2024-05-17T15:12:11.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 410278912 bytes
2024-05-17T15:12:11.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 488737792 bytes
2024-05-17T15:12:12.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 520678400 bytes
2024-05-17T15:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 535896064 bytes
2024-05-17T15:12:12.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 593355776 bytes
2024-05-17T15:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 657633280 bytes
2024-05-17T15:12:12.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 658253824 bytes
2024-05-17T15:12:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 696394752 bytes
2024-05-17T15:12:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 702867456 bytes
2024-05-17T15:12:13.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 742745088 bytes
2024-05-17T15:12:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 747527168 bytes
2024-05-17T15:12:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 764905472 bytes
2024-05-17T15:12:13.682 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 818125824 bytes
2024-05-17T15:12:13.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 888807424 bytes
2024-05-17T15:12:14.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 894616576 bytes
2024-05-17T15:12:14.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 918145024 bytes
2024-05-17T15:12:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 918739968 bytes
2024-05-17T15:12:14.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 952049664 bytes
2024-05-17T15:12:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 998059008 bytes
2024-05-17T15:12:14.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1018363904 bytes
2024-05-17T15:12:14.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1060246528 bytes
2024-05-17T15:12:15.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1134471168 bytes
2024-05-17T15:12:15.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1156780032 bytes
2024-05-17T15:12:15.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1223530496 bytes
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_25
2024-05-17T15:12:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_25 128M
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_25
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_25 snapshot
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:12:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot
2024-05-17T15:12:16.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:16.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_25
2024-05-17T15:12:16.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_25':
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13a3ac72d0d7
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13a3ac72d0d7
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:15 2024
2024-05-17T15:12:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:15 2024
2024-05-17T15:12:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:15 2024
2024-05-17T15:12:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 256ee516-bc3e-401b-8783-9eec22739431
2024-05-17T15:12:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:16.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:16.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_25
2024-05-17T15:12:16.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:16.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:16.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:12:16.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:16.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=5
2024-05-17T15:12:16.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:16.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 64506880 bytes
2024-05-17T15:12:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 149055488 bytes
2024-05-17T15:12:17.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 206943232 bytes
2024-05-17T15:12:17.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 252125184 bytes
2024-05-17T15:12:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 293410816 bytes
2024-05-17T15:12:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 335467520 bytes
2024-05-17T15:12:18.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 381415424 bytes
2024-05-17T15:12:18.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 404962304 bytes
2024-05-17T15:12:18.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 435897344 bytes
2024-05-17T15:12:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 517109760 bytes
2024-05-17T15:12:19.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 545641472 bytes
2024-05-17T15:12:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 566097920 bytes
2024-05-17T15:12:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 601860096 bytes
2024-05-17T15:12:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 637743104 bytes
2024-05-17T15:12:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 692742144 bytes
2024-05-17T15:12:19.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 716363776 bytes
2024-05-17T15:12:20.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 748015616 bytes
2024-05-17T15:12:20.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 766563328 bytes
2024-05-17T15:12:20.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 781777920 bytes
2024-05-17T15:12:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 873384960 bytes
2024-05-17T15:12:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 941706240 bytes
2024-05-17T15:12:21.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1029484544 bytes
2024-05-17T15:12:21.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1040262144 bytes
2024-05-17T15:12:21.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1111127040 bytes
2024-05-17T15:12:21.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:21.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:21.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:21.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_26
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:12:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_26 128M
2024-05-17T15:12:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:21.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:21.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_26
2024-05-17T15:12:21.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:21.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_26 snapshot
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot
2024-05-17T15:12:22.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_26
2024-05-17T15:12:22.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_26':
2024-05-17T15:12:22.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:22.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:22.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:22.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13af188946fd
2024-05-17T15:12:22.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13af188946fd
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:21 2024
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:21 2024
2024-05-17T15:12:22.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:21 2024
2024-05-17T15:12:22.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:22.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:22.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 798e27a4-f2bd-44ea-ac4a-5295e0c6ebb8
2024-05-17T15:12:22.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_26
2024-05-17T15:12:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:12:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:22.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:12:22.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:22.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:23.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 108631040 bytes
2024-05-17T15:12:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 149319680 bytes
2024-05-17T15:12:23.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 207306752 bytes
2024-05-17T15:12:23.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 225367040 bytes
2024-05-17T15:12:23.992 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 315499520 bytes
2024-05-17T15:12:24.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 328096768 bytes
2024-05-17T15:12:24.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 401439744 bytes
2024-05-17T15:12:24.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 432812032 bytes
2024-05-17T15:12:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 496531456 bytes
2024-05-17T15:12:24.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 526707712 bytes
2024-05-17T15:12:25.036 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0
2024-05-17T15:12:25.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15
2024-05-17T15:12:25.037 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons
2024-05-17T15:12:25.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T15:12:25.039+0000 7f5528d98640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 38888) UID: 1000
2024-05-17T15:12:25.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 621025280 bytes
2024-05-17T15:12:25.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 649026560 bytes
2024-05-17T15:12:25.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 693188608 bytes
2024-05-17T15:12:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 758084608 bytes
2024-05-17T15:12:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 833819648 bytes
2024-05-17T15:12:26.325 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 887717888 bytes
2024-05-17T15:12:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 949269504 bytes
2024-05-17T15:12:26.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 972662784 bytes
2024-05-17T15:12:27.038 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0
2024-05-17T15:12:27.038 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:12:27.039 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0
2024-05-17T15:12:27.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon
2024-05-17T15:12:27.039 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0
2024-05-17T15:12:27.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started
2024-05-17T15:12:27.042 INFO:tasks.rbd_mirror_thrash:waiting for 99 secs before thrashing
2024-05-17T15:12:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 994338816 bytes
2024-05-17T15:12:27.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 1071449088 bytes
2024-05-17T15:12:27.501 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1092579328 bytes
2024-05-17T15:12:27.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1128038400 bytes
2024-05-17T15:12:27.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1128665088 bytes
2024-05-17T15:12:28.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1182927872 bytes
2024-05-17T15:12:28.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1240286208 bytes
2024-05-17T15:12:28.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1271282688 bytes
2024-05-17T15:12:28.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1278001152 bytes
2024-05-17T15:12:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1282835456 bytes
2024-05-17T15:12:28.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1328372736 bytes
2024-05-17T15:12:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1395303424 bytes
2024-05-17T15:12:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1480584192 bytes
2024-05-17T15:12:29.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1481178112 bytes
2024-05-17T15:12:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1493023744 bytes
2024-05-17T15:12:29.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:29.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_27
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_27 128M
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_27
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_27 snapshot
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:29.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:12:29.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:29.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot
2024-05-17T15:12:30.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_27
2024-05-17T15:12:30.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_27':
2024-05-17T15:12:30.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13c17627c293
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13c17627c293
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:29 2024
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:29 2024
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:29 2024
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:30.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 88d5ad35-b4ea-4cf0-968b-4a1d374fed30
2024-05-17T15:12:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:30.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:30.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_27
2024-05-17T15:12:30.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:30.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:30.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:12:30.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:30.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:12:30.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:30.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:31.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 30121984 bytes
2024-05-17T15:12:31.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 47418368 bytes
2024-05-17T15:12:31.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 73559040 bytes
2024-05-17T15:12:31.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 133030912 bytes
2024-05-17T15:12:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 228548608 bytes
2024-05-17T15:12:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 247164928 bytes
2024-05-17T15:12:32.289 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 337561600 bytes
2024-05-17T15:12:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 430152704 bytes
2024-05-17T15:12:32.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 465316864 bytes
2024-05-17T15:12:32.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 478611456 bytes
2024-05-17T15:12:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 530480128 bytes
2024-05-17T15:12:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 556531712 bytes
2024-05-17T15:12:33.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 557111296 bytes
2024-05-17T15:12:33.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 584487936 bytes
2024-05-17T15:12:34.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 678797312 bytes
2024-05-17T15:12:34.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 736826368 bytes
2024-05-17T15:12:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 782828544 bytes
2024-05-17T15:12:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 846205952 bytes
2024-05-17T15:12:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 885669888 bytes
2024-05-17T15:12:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 949307392 bytes
2024-05-17T15:12:35.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 996166656 bytes
2024-05-17T15:12:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1053593600 bytes
2024-05-17T15:12:35.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1115071488 bytes
2024-05-17T15:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1194940416 bytes
2024-05-17T15:12:36.168 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1216627712 bytes
2024-05-17T15:12:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1304816640 bytes
2024-05-17T15:12:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1351521280 bytes
2024-05-17T15:12:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1357364224 bytes
2024-05-17T15:12:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1359113216 bytes
2024-05-17T15:12:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1402474496 bytes
2024-05-17T15:12:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1405010944 bytes
2024-05-17T15:12:37.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1408752640 bytes
2024-05-17T15:12:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1459789824 bytes
2024-05-17T15:12:37.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1512549376 bytes
2024-05-17T15:12:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1528162304 bytes
2024-05-17T15:12:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1544457216 bytes
2024-05-17T15:12:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1621496832 bytes
2024-05-17T15:12:38.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1637135360 bytes
2024-05-17T15:12:38.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1666509824 bytes
2024-05-17T15:12:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1725731840 bytes
2024-05-17T15:12:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1772166144 bytes
2024-05-17T15:12:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1772805120 bytes
2024-05-17T15:12:39.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1819189248 bytes
2024-05-17T15:12:39.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1822600192 bytes
2024-05-17T15:12:39.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1841269760 bytes
2024-05-17T15:12:39.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1873100800 bytes
2024-05-17T15:12:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1913530368 bytes
2024-05-17T15:12:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1984902144 bytes
2024-05-17T15:12:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2078145536 bytes
2024-05-17T15:12:40.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2116390912 bytes
2024-05-17T15:12:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2153459712 bytes
2024-05-17T15:12:41.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2215465984 bytes
2024-05-17T15:12:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2257335296 bytes
2024-05-17T15:12:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2309515264 bytes
2024-05-17T15:12:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2362827776 bytes
2024-05-17T15:12:41.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2384011264 bytes
2024-05-17T15:12:42.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2433037312 bytes
2024-05-17T15:12:42.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2433652736 bytes
2024-05-17T15:12:42.330 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2455301120 bytes
2024-05-17T15:12:42.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2519775232 bytes
2024-05-17T15:12:42.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2568787968 bytes
2024-05-17T15:12:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2608076800 bytes
2024-05-17T15:12:43.029 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9645966818109223) > weight (0.25)
2024-05-17T15:12:43.030 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1
2024-05-17T15:12:43.030 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15
2024-05-17T15:12:43.030 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons
2024-05-17T15:12:43.031 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi171.stderr:2024-05-17T15:12:43.031+0000 7f59df398640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1  (PID: 29320) UID: 1000
2024-05-17T15:12:43.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2644653056 bytes
2024-05-17T15:12:43.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2672300032 bytes
2024-05-17T15:12:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2730475520 bytes
2024-05-17T15:12:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2775520256 bytes
2024-05-17T15:12:43.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_28
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_28 128M
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_28
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_28 snapshot
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:43.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:12:43.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:43.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot
2024-05-17T15:12:44.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_28
2024-05-17T15:12:44.439 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_28':
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13ccf84285a6
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13ccf84285a6
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:43 2024
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:43 2024
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:43 2024
2024-05-17T15:12:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 3fef99bd-821a-4e44-b224-1260889e1805
2024-05-17T15:12:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_28
2024-05-17T15:12:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:44.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:12:44.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:44.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=5
2024-05-17T15:12:44.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:44.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 8032256 bytes
2024-05-17T15:12:44.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 8657920 bytes
2024-05-17T15:12:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 54265856 bytes
2024-05-17T15:12:45.031 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1
2024-05-17T15:12:45.031 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:12:45.031 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1
2024-05-17T15:12:45.032 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon
2024-05-17T15:12:45.032 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1
2024-05-17T15:12:45.036 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started
2024-05-17T15:12:45.036 INFO:tasks.rbd_mirror_thrash:waiting for 69 secs before thrashing
2024-05-17T15:12:45.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 103247872 bytes
2024-05-17T15:12:45.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 128496640 bytes
2024-05-17T15:12:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 188603392 bytes
2024-05-17T15:12:45.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 252437504 bytes
2024-05-17T15:12:46.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 332299264 bytes
2024-05-17T15:12:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 396557312 bytes
2024-05-17T15:12:46.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 406249472 bytes
2024-05-17T15:12:46.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 441655296 bytes
2024-05-17T15:12:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 442270720 bytes
2024-05-17T15:12:46.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 461172736 bytes
2024-05-17T15:12:47.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 515491840 bytes
2024-05-17T15:12:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 577543168 bytes
2024-05-17T15:12:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 620779520 bytes
2024-05-17T15:12:47.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 642196480 bytes
2024-05-17T15:12:47.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 708253696 bytes
2024-05-17T15:12:48.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 756546560 bytes
2024-05-17T15:12:48.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 788525056 bytes
2024-05-17T15:12:48.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 827807744 bytes
2024-05-17T15:12:48.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 828409856 bytes
2024-05-17T15:12:48.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 888894464 bytes
2024-05-17T15:12:48.937 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 931110912 bytes
2024-05-17T15:12:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 986926080 bytes
2024-05-17T15:12:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1036805120 bytes
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_29
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_29 128M
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:49.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_29
2024-05-17T15:12:49.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:49.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:49.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_29 snapshot
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:49.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot
2024-05-17T15:12:50.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:12:50.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_29
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_29':
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13e0bf676bd3
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13e0bf676bd3
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:12:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:49 2024
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:49 2024
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:49 2024
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7fda8009-e76c-42ba-8a4f-b0fc43ecf4b6
2024-05-17T15:12:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:12:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:12:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_29
2024-05-17T15:12:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:12:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:12:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=9
2024-05-17T15:12:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:12:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:12:50.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 123862016 bytes
2024-05-17T15:12:51.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 144030720 bytes
2024-05-17T15:12:51.260 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 166735872 bytes
2024-05-17T15:12:51.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 249332736 bytes
2024-05-17T15:12:51.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 313512960 bytes
2024-05-17T15:12:51.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 332683264 bytes
2024-05-17T15:12:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 378657792 bytes
2024-05-17T15:12:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 406499328 bytes
2024-05-17T15:12:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 426885120 bytes
2024-05-17T15:12:52.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 534545408 bytes
2024-05-17T15:12:52.913 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 560708608 bytes
2024-05-17T15:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 616283136 bytes
2024-05-17T15:12:53.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 637967360 bytes
2024-05-17T15:12:53.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 654152704 bytes
2024-05-17T15:12:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 670717952 bytes
2024-05-17T15:12:53.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 690918400 bytes
2024-05-17T15:12:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 706934784 bytes
2024-05-17T15:12:54.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 778094592 bytes
2024-05-17T15:12:54.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 806308864 bytes
2024-05-17T15:12:54.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 859985920 bytes
2024-05-17T15:12:54.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 899270656 bytes
2024-05-17T15:12:54.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 946165760 bytes
2024-05-17T15:12:55.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 999690240 bytes
2024-05-17T15:12:55.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1042217984 bytes
2024-05-17T15:12:55.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1120499712 bytes
2024-05-17T15:12:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1217392640 bytes
2024-05-17T15:12:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1243172864 bytes
2024-05-17T15:12:56.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1283383296 bytes
2024-05-17T15:12:56.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1340162048 bytes
2024-05-17T15:12:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1369688064 bytes
2024-05-17T15:12:56.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1434867712 bytes
2024-05-17T15:12:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1489287168 bytes
2024-05-17T15:12:57.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1523819520 bytes
2024-05-17T15:12:57.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1614325760 bytes
2024-05-17T15:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1694710784 bytes
2024-05-17T15:12:58.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1737128960 bytes
2024-05-17T15:12:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1773008896 bytes
2024-05-17T15:12:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1784331264 bytes
2024-05-17T15:12:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1817193472 bytes
2024-05-17T15:12:58.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1917550592 bytes
2024-05-17T15:12:59.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1987707904 bytes
2024-05-17T15:12:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 2039330816 bytes
2024-05-17T15:12:59.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 2052719616 bytes
2024-05-17T15:12:59.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_30
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_30 128M
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:12:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:12:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:12:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_30
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_30 snapshot
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:12:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot
2024-05-17T15:13:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_30
2024-05-17T15:13:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_30':
2024-05-17T15:13:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13ecde7e095c
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13ecde7e095c
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:59 2024
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:59 2024
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:59 2024
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: b890af64-5f4d-4763-9ed5-689fae1e3fdc
2024-05-17T15:13:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:00.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:00.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_30
2024-05-17T15:13:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:13:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:13:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:00.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 60099584 bytes
2024-05-17T15:13:01.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 74413056 bytes
2024-05-17T15:13:01.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 114092032 bytes
2024-05-17T15:13:01.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 161528832 bytes
2024-05-17T15:13:01.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 215926784 bytes
2024-05-17T15:13:01.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 237760512 bytes
2024-05-17T15:13:01.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 261810176 bytes
2024-05-17T15:13:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 301527040 bytes
2024-05-17T15:13:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 343400448 bytes
2024-05-17T15:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 382305280 bytes
2024-05-17T15:13:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 414135296 bytes
2024-05-17T15:13:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 481864704 bytes
2024-05-17T15:13:03.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 520633344 bytes
2024-05-17T15:13:03.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 618503168 bytes
2024-05-17T15:13:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 661624832 bytes
2024-05-17T15:13:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 735983616 bytes
2024-05-17T15:13:04.149 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 809333760 bytes
2024-05-17T15:13:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 828366848 bytes
2024-05-17T15:13:04.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 861062144 bytes
2024-05-17T15:13:04.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 903170048 bytes
2024-05-17T15:13:04.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 937507840 bytes
2024-05-17T15:13:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 961676288 bytes
2024-05-17T15:13:05.154 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 986284032 bytes
2024-05-17T15:13:05.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1034072064 bytes
2024-05-17T15:13:05.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1092018176 bytes
2024-05-17T15:13:05.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1101001728 bytes
2024-05-17T15:13:05.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1129507840 bytes
2024-05-17T15:13:06.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1211404288 bytes
2024-05-17T15:13:06.284 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1232766976 bytes
2024-05-17T15:13:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1250065408 bytes
2024-05-17T15:13:06.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1272520704 bytes
2024-05-17T15:13:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1298435072 bytes
2024-05-17T15:13:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1427928064 bytes
2024-05-17T15:13:07.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1469857792 bytes
2024-05-17T15:13:07.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1478262784 bytes
2024-05-17T15:13:07.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1523302400 bytes
2024-05-17T15:13:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1556549632 bytes
2024-05-17T15:13:07.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1557165056 bytes
2024-05-17T15:13:08.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1703133184 bytes
2024-05-17T15:13:08.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1778349056 bytes
2024-05-17T15:13:08.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1797835776 bytes
2024-05-17T15:13:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1854690304 bytes
2024-05-17T15:13:09.163 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1921156096 bytes
2024-05-17T15:13:09.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1946636288 bytes
2024-05-17T15:13:09.453 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1953955840 bytes
2024-05-17T15:13:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2016199680 bytes
2024-05-17T15:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2059546624 bytes
2024-05-17T15:13:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2105834496 bytes
2024-05-17T15:13:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2174579712 bytes
2024-05-17T15:13:10.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2219205632 bytes
2024-05-17T15:13:10.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2291539968 bytes
2024-05-17T15:13:10.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2316521472 bytes
2024-05-17T15:13:11.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2361158656 bytes
2024-05-17T15:13:11.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2419788800 bytes
2024-05-17T15:13:11.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2455883776 bytes
2024-05-17T15:13:11.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_31
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_31 128M
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:13:11.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:11.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_31
2024-05-17T15:13:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:13:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_31 snapshot
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot
2024-05-17T15:13:12.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_31
2024-05-17T15:13:12.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_31':
2024-05-17T15:13:12.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:12.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:12.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 13f8f7463efe
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.13f8f7463efe
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:11 2024
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:11 2024
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:11 2024
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 2a498d86-9a3f-4036-b490-785be695571e
2024-05-17T15:13:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_31
2024-05-17T15:13:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:13:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=9
2024-05-17T15:13:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 20550656 bytes
2024-05-17T15:13:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 36724736 bytes
2024-05-17T15:13:13.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 48594944 bytes
2024-05-17T15:13:13.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 89657344 bytes
2024-05-17T15:13:13.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 198667264 bytes
2024-05-17T15:13:13.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 245854208 bytes
2024-05-17T15:13:14.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 285048832 bytes
2024-05-17T15:13:14.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 297849856 bytes
2024-05-17T15:13:14.465 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 350984192 bytes
2024-05-17T15:13:14.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 363253760 bytes
2024-05-17T15:13:14.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 363891712 bytes
2024-05-17T15:13:14.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 403270656 bytes
2024-05-17T15:13:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 403908608 bytes
2024-05-17T15:13:15.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 440391680 bytes
2024-05-17T15:13:15.402 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 483565568 bytes
2024-05-17T15:13:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 536715264 bytes
2024-05-17T15:13:15.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 574002176 bytes
2024-05-17T15:13:16.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 669747200 bytes
2024-05-17T15:13:16.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 718012416 bytes
2024-05-17T15:13:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 733907968 bytes
2024-05-17T15:13:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 781814784 bytes
2024-05-17T15:13:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 854714368 bytes
2024-05-17T15:13:17.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 876176384 bytes
2024-05-17T15:13:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 913417216 bytes
2024-05-17T15:13:17.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 992571392 bytes
2024-05-17T15:13:17.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1034561536 bytes
2024-05-17T15:13:17.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1047439360 bytes
2024-05-17T15:13:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1101877248 bytes
2024-05-17T15:13:18.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1211064320 bytes
2024-05-17T15:13:18.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1211671552 bytes
2024-05-17T15:13:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1231691776 bytes
2024-05-17T15:13:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1242054656 bytes
2024-05-17T15:13:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1301007360 bytes
2024-05-17T15:13:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1315921920 bytes
2024-05-17T15:13:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1339537408 bytes
2024-05-17T15:13:19.457 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1371894784 bytes
2024-05-17T15:13:19.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1435979776 bytes
2024-05-17T15:13:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1505747968 bytes
2024-05-17T15:13:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1625709568 bytes
2024-05-17T15:13:20.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1664257024 bytes
2024-05-17T15:13:20.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1676634112 bytes
2024-05-17T15:13:20.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1682994176 bytes
2024-05-17T15:13:20.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1739758592 bytes
2024-05-17T15:13:21.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1755736064 bytes
2024-05-17T15:13:21.325 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1816532992 bytes
2024-05-17T15:13:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1827108864 bytes
2024-05-17T15:13:21.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1827710976 bytes
2024-05-17T15:13:21.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:13:21.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:13:21.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:13:21.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:21.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:13:21.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_32
2024-05-17T15:13:21.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M
2024-05-17T15:13:21.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:21.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_32 128M
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:13:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:13:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_32
2024-05-17T15:13:21.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:13:21.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:21.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_32 snapshot
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot
2024-05-17T15:13:22.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_32
2024-05-17T15:13:22.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_32':
2024-05-17T15:13:22.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 140423b8dc98
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.140423b8dc98
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:21 2024
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:21 2024
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:21 2024
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ebb6c97f-7a9a-4158-a875-14c29371e6d8
2024-05-17T15:13:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_32
2024-05-17T15:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 45659136 bytes
2024-05-17T15:13:23.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 98699264 bytes
2024-05-17T15:13:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 179357696 bytes
2024-05-17T15:13:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 221906944 bytes
2024-05-17T15:13:24.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 274870272 bytes
2024-05-17T15:13:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 285721600 bytes
2024-05-17T15:13:24.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 379771904 bytes
2024-05-17T15:13:24.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 383840256 bytes
2024-05-17T15:13:24.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 465149952 bytes
2024-05-17T15:13:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 521730048 bytes
2024-05-17T15:13:25.231 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 556961792 bytes
2024-05-17T15:13:25.452 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 607425536 bytes
2024-05-17T15:13:25.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 713796608 bytes
2024-05-17T15:13:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 726999040 bytes
2024-05-17T15:13:26.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 727616512 bytes
2024-05-17T15:13:26.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 780759040 bytes
2024-05-17T15:13:26.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 782320640 bytes
2024-05-17T15:13:26.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 815008768 bytes
2024-05-17T15:13:26.898 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 840980480 bytes
2024-05-17T15:13:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 861527040 bytes
2024-05-17T15:13:27.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 927690752 bytes
2024-05-17T15:13:27.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 987104256 bytes
2024-05-17T15:13:27.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1025055744 bytes
2024-05-17T15:13:27.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1093813248 bytes
2024-05-17T15:13:28.213 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1162903552 bytes
2024-05-17T15:13:28.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1227673600 bytes
2024-05-17T15:13:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1289658368 bytes
2024-05-17T15:13:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1336393728 bytes
2024-05-17T15:13:29.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1356486656 bytes
2024-05-17T15:13:29.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1415768064 bytes
2024-05-17T15:13:29.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1446317056 bytes
2024-05-17T15:13:29.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1451951104 bytes
2024-05-17T15:13:29.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_33
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:13:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_33 128M
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:13:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:13:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_33
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_33 snapshot
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:29.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:13:29.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:29.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot
2024-05-17T15:13:30.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:30.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_33
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_33':
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1410f423d763
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1410f423d763
2024-05-17T15:13:30.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:29 2024
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:29 2024
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:29 2024
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: b26f73bc-22d7-4d4a-b487-999e05b95f20
2024-05-17T15:13:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:30.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:30.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_33
2024-05-17T15:13:30.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:30.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:30.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:13:30.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:30.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=5
2024-05-17T15:13:30.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:30.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:30.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 25732096 bytes
2024-05-17T15:13:30.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 71699456 bytes
2024-05-17T15:13:31.155 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 144950272 bytes
2024-05-17T15:13:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 163077120 bytes
2024-05-17T15:13:31.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 227397632 bytes
2024-05-17T15:13:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 245778432 bytes
2024-05-17T15:13:31.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 269272064 bytes
2024-05-17T15:13:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 379371520 bytes
2024-05-17T15:13:32.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 421769216 bytes
2024-05-17T15:13:32.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 463690752 bytes
2024-05-17T15:13:32.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 484696064 bytes
2024-05-17T15:13:32.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 503997440 bytes
2024-05-17T15:13:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 556675072 bytes
2024-05-17T15:13:33.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 631119872 bytes
2024-05-17T15:13:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 648354816 bytes
2024-05-17T15:13:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 687458304 bytes
2024-05-17T15:13:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 703257600 bytes
2024-05-17T15:13:34.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 732483584 bytes
2024-05-17T15:13:34.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 781789184 bytes
2024-05-17T15:13:34.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 782372864 bytes
2024-05-17T15:13:34.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 832499712 bytes
2024-05-17T15:13:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 870085632 bytes
2024-05-17T15:13:35.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 896471040 bytes
2024-05-17T15:13:35.293 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 941258752 bytes
2024-05-17T15:13:35.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:13:35.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:13:35.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:13:35.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:35.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:13:35.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_34
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_34
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_34 128M
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_34
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_34
2024-05-17T15:13:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_34 snapshot
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_34
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot
2024-05-17T15:13:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_34
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_34':
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 141c3fa2232e
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.141c3fa2232e
2024-05-17T15:13:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:35 2024
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:35 2024
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:35 2024
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ce4d581a-c117-4cb9-95bc-06b7949fe51a
2024-05-17T15:13:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:36.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:36.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_34
2024-05-17T15:13:36.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:36.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:36.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_34
2024-05-17T15:13:36.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:36.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:13:36.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:36.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:36.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 42454016 bytes
2024-05-17T15:13:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 78727168 bytes
2024-05-17T15:13:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 122457088 bytes
2024-05-17T15:13:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 135435264 bytes
2024-05-17T15:13:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 153703424 bytes
2024-05-17T15:13:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 171452416 bytes
2024-05-17T15:13:37.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 189074432 bytes
2024-05-17T15:13:37.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 239235072 bytes
2024-05-17T15:13:38.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 278535168 bytes
2024-05-17T15:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 358210560 bytes
2024-05-17T15:13:38.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 382699520 bytes
2024-05-17T15:13:38.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 383311872 bytes
2024-05-17T15:13:39.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 440034304 bytes
2024-05-17T15:13:39.191 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 461039616 bytes
2024-05-17T15:13:39.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 522017792 bytes
2024-05-17T15:13:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 539810816 bytes
2024-05-17T15:13:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 554850304 bytes
2024-05-17T15:13:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 581925888 bytes
2024-05-17T15:13:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 582538240 bytes
2024-05-17T15:13:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 615385088 bytes
2024-05-17T15:13:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 636252160 bytes
2024-05-17T15:13:40.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 688939008 bytes
2024-05-17T15:13:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 700713984 bytes
2024-05-17T15:13:40.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 762244096 bytes
2024-05-17T15:13:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 795595776 bytes
2024-05-17T15:13:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 833593344 bytes
2024-05-17T15:13:41.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 935895040 bytes
2024-05-17T15:13:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1015187456 bytes
2024-05-17T15:13:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1036566528 bytes
2024-05-17T15:13:42.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1062419456 bytes
2024-05-17T15:13:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1114176512 bytes
2024-05-17T15:13:42.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1143977984 bytes
2024-05-17T15:13:42.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1213310976 bytes
2024-05-17T15:13:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1266360320 bytes
2024-05-17T15:13:43.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1291245568 bytes
2024-05-17T15:13:43.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1317828608 bytes
2024-05-17T15:13:43.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1372404736 bytes
2024-05-17T15:13:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1438183424 bytes
2024-05-17T15:13:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1503008768 bytes
2024-05-17T15:13:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1560375296 bytes
2024-05-17T15:13:44.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1583915008 bytes
2024-05-17T15:13:44.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1638614016 bytes
2024-05-17T15:13:44.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1662627840 bytes
2024-05-17T15:13:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1718537216 bytes
2024-05-17T15:13:45.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1738758144 bytes
2024-05-17T15:13:45.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1739347968 bytes
2024-05-17T15:13:45.409 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1774649344 bytes
2024-05-17T15:13:45.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1783075840 bytes
2024-05-17T15:13:45.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1807104000 bytes
2024-05-17T15:13:45.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 1860396032 bytes
2024-05-17T15:13:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 1919682560 bytes
2024-05-17T15:13:46.287 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 1941566464 bytes
2024-05-17T15:13:46.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 1951578112 bytes
2024-05-17T15:13:46.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 1976354816 bytes
2024-05-17T15:13:46.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2057392128 bytes
2024-05-17T15:13:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2091731968 bytes
2024-05-17T15:13:47.209 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2133232640 bytes
2024-05-17T15:13:47.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2153699328 bytes
2024-05-17T15:13:47.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_35
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_35
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_35 128M
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_35
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_35
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_35 snapshot
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_35
2024-05-17T15:13:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot
2024-05-17T15:13:48.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:48.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_35
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_35':
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1428602c8296
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1428602c8296
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:47 2024
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:47 2024
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:47 2024
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7dfdf998-4021-4cc6-b062-bea048b502b5
2024-05-17T15:13:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:48.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:48.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_35
2024-05-17T15:13:48.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:48.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:48.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_35
2024-05-17T15:13:48.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:48.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=5
2024-05-17T15:13:48.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:48.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 43480064 bytes
2024-05-17T15:13:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 129562624 bytes
2024-05-17T15:13:49.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 171136000 bytes
2024-05-17T15:13:49.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 174816256 bytes
2024-05-17T15:13:49.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 222976000 bytes
2024-05-17T15:13:49.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 304147456 bytes
2024-05-17T15:13:50.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 403977216 bytes
2024-05-17T15:13:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 419728384 bytes
2024-05-17T15:13:50.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 433769472 bytes
2024-05-17T15:13:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 434398208 bytes
2024-05-17T15:13:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 480161792 bytes
2024-05-17T15:13:50.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 505714688 bytes
2024-05-17T15:13:51.060 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 551183360 bytes
2024-05-17T15:13:51.235 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 612703232 bytes
2024-05-17T15:13:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 613320704 bytes
2024-05-17T15:13:51.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 682718208 bytes
2024-05-17T15:13:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 802616320 bytes
2024-05-17T15:13:52.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 846472192 bytes
2024-05-17T15:13:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 898666496 bytes
2024-05-17T15:13:52.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 944107520 bytes
2024-05-17T15:13:52.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1010567168 bytes
2024-05-17T15:13:53.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1112376320 bytes
2024-05-17T15:13:53.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1148979200 bytes
2024-05-17T15:13:53.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_36
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_36
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_36 128M
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_36
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:13:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_36
2024-05-17T15:13:53.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:13:53.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_36 snapshot
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_36
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:13:53.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot
2024-05-17T15:13:54.037 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7404360120388048) > weight (0.25)
2024-05-17T15:13:54.038 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9649450893274333) > weight (0.25)
2024-05-17T15:13:54.038 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T15:13:54.038 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T15:13:54.039 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons
2024-05-17T15:13:54.040 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T15:13:54.037+0000 7f57b622b640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 37466) UID: 1000
2024-05-17T15:13:54.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:13:54.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_36
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_36':
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 143498f32c06
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.143498f32c06
2024-05-17T15:13:54.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:53 2024
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:53 2024
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:53 2024
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 103e3a70-c1ac-412c-8531-956db0571fac
2024-05-17T15:13:54.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:13:54.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:13:54.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_36
2024-05-17T15:13:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:13:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:13:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_36
2024-05-17T15:13:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:13:54.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:13:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:13:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:13:54.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 55406592 bytes
2024-05-17T15:13:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 100184064 bytes
2024-05-17T15:13:55.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 116435968 bytes
2024-05-17T15:13:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 152604672 bytes
2024-05-17T15:13:55.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 186901504 bytes
2024-05-17T15:13:55.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 212499456 bytes
2024-05-17T15:13:56.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 309380096 bytes
2024-05-17T15:13:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 364479488 bytes
2024-05-17T15:13:56.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 422192128 bytes
2024-05-17T15:13:56.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 540148736 bytes
2024-05-17T15:13:57.032 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 576815104 bytes
2024-05-17T15:13:57.318 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 657491968 bytes
2024-05-17T15:13:57.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 714958848 bytes
2024-05-17T15:13:57.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 715578368 bytes
2024-05-17T15:13:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 740955136 bytes
2024-05-17T15:13:57.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 758722560 bytes
2024-05-17T15:13:58.041 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T15:13:58.041 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:13:58.042 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T15:13:58.042 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T15:13:58.042 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T15:13:58.047 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T15:13:58.047 INFO:tasks.rbd_mirror_thrash:waiting for 118 secs before thrashing
2024-05-17T15:13:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 770041856 bytes
2024-05-17T15:13:58.384 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 826577920 bytes
2024-05-17T15:13:58.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 857444352 bytes
2024-05-17T15:13:58.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 871501824 bytes
2024-05-17T15:13:58.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 911198208 bytes
2024-05-17T15:13:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 913792000 bytes
2024-05-17T15:13:59.204 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 951659520 bytes
2024-05-17T15:13:59.348 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 969418752 bytes
2024-05-17T15:13:59.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1010262016 bytes
2024-05-17T15:13:59.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1036116992 bytes
2024-05-17T15:13:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1071057920 bytes
2024-05-17T15:14:00.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1104106496 bytes
2024-05-17T15:14:00.323 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1154945024 bytes
2024-05-17T15:14:00.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1200927744 bytes
2024-05-17T15:14:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1232167936 bytes
2024-05-17T15:14:00.846 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1263049728 bytes
2024-05-17T15:14:01.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1281446912 bytes
2024-05-17T15:14:01.260 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1345705984 bytes
2024-05-17T15:14:01.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1366398976 bytes
2024-05-17T15:14:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1438728192 bytes
2024-05-17T15:14:01.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1496541184 bytes
2024-05-17T15:14:02.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1529409536 bytes
2024-05-17T15:14:02.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1603771392 bytes
2024-05-17T15:14:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1659589632 bytes
2024-05-17T15:14:02.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1660217344 bytes
2024-05-17T15:14:03.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1710838784 bytes
2024-05-17T15:14:03.271 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1786821632 bytes
2024-05-17T15:14:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1798538240 bytes
2024-05-17T15:14:03.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1852661760 bytes
2024-05-17T15:14:03.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1920500736 bytes
2024-05-17T15:14:04.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1970618368 bytes
2024-05-17T15:14:04.249 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2006493184 bytes
2024-05-17T15:14:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2031048704 bytes
2024-05-17T15:14:04.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2066864128 bytes
2024-05-17T15:14:04.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2114486272 bytes
2024-05-17T15:14:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2149659648 bytes
2024-05-17T15:14:05.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2258363392 bytes
2024-05-17T15:14:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2289460224 bytes
2024-05-17T15:14:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2320391168 bytes
2024-05-17T15:14:05.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2374394880 bytes
2024-05-17T15:14:06.043 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9050707609485396) > weight (0.25)
2024-05-17T15:14:06.043 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8863769620295517) > weight (0.25)
2024-05-17T15:14:06.043 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5071530965555962) > weight (0.25)
2024-05-17T15:14:06.043 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.37694568885951907) > weight (0.25)
2024-05-17T15:14:06.043 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing
2024-05-17T15:14:06.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2386096128 bytes
2024-05-17T15:14:06.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2413682688 bytes
2024-05-17T15:14:06.427 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2456968192 bytes
2024-05-17T15:14:06.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2544364544 bytes
2024-05-17T15:14:06.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2627656704 bytes
2024-05-17T15:14:07.180 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2671078400 bytes
2024-05-17T15:14:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2671668224 bytes
2024-05-17T15:14:07.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2761749504 bytes
2024-05-17T15:14:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2776085504 bytes
2024-05-17T15:14:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2781032448 bytes
2024-05-17T15:14:07.980 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2781651968 bytes
2024-05-17T15:14:08.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 2819261440 bytes
2024-05-17T15:14:08.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 2837381120 bytes
2024-05-17T15:14:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 2922593280 bytes
2024-05-17T15:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_37
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_37
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_37 128M
2024-05-17T15:14:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_37
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_37
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_37 snapshot
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:08.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_37
2024-05-17T15:14:08.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:08.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot
2024-05-17T15:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:14:09.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_37
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_37':
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1449fb008353
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1449fb008353
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:14:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:14:08 2024
2024-05-17T15:14:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:14:08 2024
2024-05-17T15:14:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:14:08 2024
2024-05-17T15:14:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:14:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:14:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7372390a-2d75-407e-969f-a846d4ccd04a
2024-05-17T15:14:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:14:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:14:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_37
2024-05-17T15:14:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_37
2024-05-17T15:14:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:14:09.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=9
2024-05-17T15:14:09.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:14:09.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:14:09.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 22331392 bytes
2024-05-17T15:14:10.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 51490816 bytes
2024-05-17T15:14:10.230 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 78193664 bytes
2024-05-17T15:14:10.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 140739584 bytes
2024-05-17T15:14:10.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 203639808 bytes
2024-05-17T15:14:11.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 347936768 bytes
2024-05-17T15:14:11.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 371818496 bytes
2024-05-17T15:14:11.388 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 383569920 bytes
2024-05-17T15:14:11.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 427186176 bytes
2024-05-17T15:14:11.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 534724608 bytes
2024-05-17T15:14:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 559913984 bytes
2024-05-17T15:14:12.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 585449472 bytes
2024-05-17T15:14:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 634862592 bytes
2024-05-17T15:14:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 657383424 bytes
2024-05-17T15:14:13.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 759599104 bytes
2024-05-17T15:14:13.158 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 766638080 bytes
2024-05-17T15:14:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 784667648 bytes
2024-05-17T15:14:13.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 806637568 bytes
2024-05-17T15:14:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 840340480 bytes
2024-05-17T15:14:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 896435200 bytes
2024-05-17T15:14:13.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 920839168 bytes
2024-05-17T15:14:14.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1030386688 bytes
2024-05-17T15:14:14.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1107597312 bytes
2024-05-17T15:14:14.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1136953344 bytes
2024-05-17T15:14:14.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1143630848 bytes
2024-05-17T15:14:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1161274368 bytes
2024-05-17T15:14:15.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1185421312 bytes
2024-05-17T15:14:15.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1216843776 bytes
2024-05-17T15:14:15.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1240456192 bytes
2024-05-17T15:14:15.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1324207104 bytes
2024-05-17T15:14:16.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1405365248 bytes
2024-05-17T15:14:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1458010112 bytes
2024-05-17T15:14:16.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1513324544 bytes
2024-05-17T15:14:16.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1560494080 bytes
2024-05-17T15:14:16.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1641552896 bytes
2024-05-17T15:14:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1671191552 bytes
2024-05-17T15:14:17.281 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1690577920 bytes
2024-05-17T15:14:17.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1715685376 bytes
2024-05-17T15:14:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1753295872 bytes
2024-05-17T15:14:17.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1801377792 bytes
2024-05-17T15:14:18.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1852133376 bytes
2024-05-17T15:14:18.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1886353408 bytes
2024-05-17T15:14:18.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1948073984 bytes
2024-05-17T15:14:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1948685312 bytes
2024-05-17T15:14:18.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_38
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_38
2024-05-17T15:14:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_38 128M
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_38
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:14:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:14:18.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:18.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_38
2024-05-17T15:14:18.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_38 snapshot
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_38
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:18.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot
2024-05-17T15:14:19.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:14:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_38
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_38':
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1455243d1e8d
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1455243d1e8d
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:14:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:14:18 2024
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:14:18 2024
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:14:18 2024
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: f852f416-3823-4fb0-a2c4-6e314ab6a994
2024-05-17T15:14:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:14:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:14:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_38
2024-05-17T15:14:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_38
2024-05-17T15:14:19.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:14:19.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:14:19.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:14:19.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:14:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 84180992 bytes
2024-05-17T15:14:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 137882624 bytes
2024-05-17T15:14:20.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 200700928 bytes
2024-05-17T15:14:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 201331712 bytes
2024-05-17T15:14:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 280504320 bytes
2024-05-17T15:14:21.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 325132288 bytes
2024-05-17T15:14:21.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 325760000 bytes
2024-05-17T15:14:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 343773184 bytes
2024-05-17T15:14:21.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 385158144 bytes
2024-05-17T15:14:21.731 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 388699136 bytes
2024-05-17T15:14:21.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 391333888 bytes
2024-05-17T15:14:22.083 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 469479424 bytes
2024-05-17T15:14:22.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 518808576 bytes
2024-05-17T15:14:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 519409664 bytes
2024-05-17T15:14:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 531432448 bytes
2024-05-17T15:14:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 553589760 bytes
2024-05-17T15:14:22.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 578638848 bytes
2024-05-17T15:14:23.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 619766784 bytes
2024-05-17T15:14:23.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 678077440 bytes
2024-05-17T15:14:23.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 712690688 bytes
2024-05-17T15:14:23.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 747742208 bytes
2024-05-17T15:14:23.844 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 801281024 bytes
2024-05-17T15:14:23.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 801879040 bytes
2024-05-17T15:14:24.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 814414848 bytes
2024-05-17T15:14:24.265 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 839867392 bytes
2024-05-17T15:14:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 870919168 bytes
2024-05-17T15:14:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 877624320 bytes
2024-05-17T15:14:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 946329600 bytes
2024-05-17T15:14:25.082 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1021283328 bytes
2024-05-17T15:14:25.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1069907968 bytes
2024-05-17T15:14:25.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1111723008 bytes
2024-05-17T15:14:25.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1164487680 bytes
2024-05-17T15:14:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1241475072 bytes
2024-05-17T15:14:26.133 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1260407808 bytes
2024-05-17T15:14:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1322368000 bytes
2024-05-17T15:14:26.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1370140672 bytes
2024-05-17T15:14:26.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1399101440 bytes
2024-05-17T15:14:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:14:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:14:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:14:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_39
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_39
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_39 128M
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_39
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_39
2024-05-17T15:14:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_39 snapshot
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_39
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot
2024-05-17T15:14:27.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:14:27.819 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_39
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_39':
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1461af079710
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1461af079710
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:14:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:14:26 2024
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:14:26 2024
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:14:26 2024
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: a7aff707-0469-4c4e-95aa-8300434ed7bc
2024-05-17T15:14:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:14:27.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:14:27.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_39
2024-05-17T15:14:27.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:27.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:27.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_39
2024-05-17T15:14:27.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:14:27.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:14:27.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:14:27.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:14:28.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 14320640 bytes
2024-05-17T15:14:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 56514560 bytes
2024-05-17T15:14:28.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 102147072 bytes
2024-05-17T15:14:28.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 128825344 bytes
2024-05-17T15:14:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 180175872 bytes
2024-05-17T15:14:29.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 232848384 bytes
2024-05-17T15:14:29.223 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 269438976 bytes
2024-05-17T15:14:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 318907392 bytes
2024-05-17T15:14:29.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 395228160 bytes
2024-05-17T15:14:29.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 445071360 bytes
2024-05-17T15:14:30.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 445710336 bytes
2024-05-17T15:14:30.281 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 517889024 bytes
2024-05-17T15:14:30.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 560119808 bytes
2024-05-17T15:14:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 591698944 bytes
2024-05-17T15:14:30.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 611760128 bytes
2024-05-17T15:14:30.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 640108544 bytes
2024-05-17T15:14:31.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 673162240 bytes
2024-05-17T15:14:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 735952896 bytes
2024-05-17T15:14:31.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 769840128 bytes
2024-05-17T15:14:31.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 815217664 bytes
2024-05-17T15:14:32.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 874847232 bytes
2024-05-17T15:14:32.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 875440128 bytes
2024-05-17T15:14:32.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 917438464 bytes
2024-05-17T15:14:32.597 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 966964224 bytes
2024-05-17T15:14:32.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1008285696 bytes
2024-05-17T15:14:32.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1026393088 bytes
2024-05-17T15:14:33.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1063966720 bytes
2024-05-17T15:14:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1125170176 bytes
2024-05-17T15:14:33.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1164796928 bytes
2024-05-17T15:14:33.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1266191360 bytes
2024-05-17T15:14:34.063 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1299478528 bytes
2024-05-17T15:14:34.219 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1339096064 bytes
2024-05-17T15:14:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1431777280 bytes
2024-05-17T15:14:34.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1477683200 bytes
2024-05-17T15:14:34.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1533768704 bytes
2024-05-17T15:14:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1590903808 bytes
2024-05-17T15:14:35.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1591520256 bytes
2024-05-17T15:14:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1668416512 bytes
2024-05-17T15:14:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1669022720 bytes
2024-05-17T15:14:35.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1702871040 bytes
2024-05-17T15:14:36.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1729836032 bytes
2024-05-17T15:14:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1730447360 bytes
2024-05-17T15:14:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1759682560 bytes
2024-05-17T15:14:36.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1760306176 bytes
2024-05-17T15:14:36.680 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1820422144 bytes
2024-05-17T15:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1868393472 bytes
2024-05-17T15:14:37.212 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1972521984 bytes
2024-05-17T15:14:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2052512768 bytes
2024-05-17T15:14:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2053385216 bytes
2024-05-17T15:14:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2078635008 bytes
2024-05-17T15:14:37.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2112344064 bytes
2024-05-17T15:14:38.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2171224064 bytes
2024-05-17T15:14:38.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2197571584 bytes
2024-05-17T15:14:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2289974272 bytes
2024-05-17T15:14:38.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2331848704 bytes
2024-05-17T15:14:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2332496896 bytes
2024-05-17T15:14:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2351623168 bytes
2024-05-17T15:14:39.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2406386688 bytes
2024-05-17T15:14:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2435168256 bytes
2024-05-17T15:14:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2446604288 bytes
2024-05-17T15:14:40.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2574069760 bytes
2024-05-17T15:14:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2637591552 bytes
2024-05-17T15:14:40.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2675680256 bytes
2024-05-17T15:14:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2711937024 bytes
2024-05-17T15:14:40.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2753753088 bytes
2024-05-17T15:14:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:14:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:14:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:14:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:14:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_40
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_40
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_40 128M
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_40
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_40
2024-05-17T15:14:40.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_40 snapshot
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_40
2024-05-17T15:14:40.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:40.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot
2024-05-17T15:14:41.456 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:14:41.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_40
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_40':
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 146de429c29b
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.146de429c29b
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:14:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:14:40 2024
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:14:40 2024
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:14:40 2024
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 1650f3f4-7463-4c89-916c-0af7a4323a97
2024-05-17T15:14:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:14:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:14:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_40
2024-05-17T15:14:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_40
2024-05-17T15:14:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:14:41.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:14:41.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:14:41.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:14:41.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 23688192 bytes
2024-05-17T15:14:42.040 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 145290240 bytes
2024-05-17T15:14:42.204 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 154705920 bytes
2024-05-17T15:14:42.313 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:14:42.312+0000 7f905860c640 -1 reset not still connected to 0x55d269112270
2024-05-17T15:14:42.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 195858432 bytes
2024-05-17T15:14:42.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 252115968 bytes
2024-05-17T15:14:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 285403136 bytes
2024-05-17T15:14:43.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 334175232 bytes
2024-05-17T15:14:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 359901184 bytes
2024-05-17T15:14:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 406638592 bytes
2024-05-17T15:14:43.523 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:14:43.523+0000 7f075cc03640 -1 reset not still connected to 0x55ec6e9592b0
2024-05-17T15:14:43.523 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:14:43.523+0000 7f075cc03640 -1 reset not still connected to 0x55ec6e959ee0
2024-05-17T15:14:43.524 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:14:43.523+0000 7f075cc03640 -1 reset not still connected to 0x55ec6ef971e0
2024-05-17T15:14:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 444310528 bytes
2024-05-17T15:14:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 481182720 bytes
2024-05-17T15:14:43.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 534511616 bytes
2024-05-17T15:14:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 591455232 bytes
2024-05-17T15:14:44.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 643000320 bytes
2024-05-17T15:14:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 708216832 bytes
2024-05-17T15:14:44.799 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 728183808 bytes
2024-05-17T15:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 799284224 bytes
2024-05-17T15:14:45.210 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 827379712 bytes
2024-05-17T15:14:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 861455360 bytes
2024-05-17T15:14:45.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 919240704 bytes
2024-05-17T15:14:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 942683136 bytes
2024-05-17T15:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 958711808 bytes
2024-05-17T15:14:46.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 996125696 bytes
2024-05-17T15:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1047373824 bytes
2024-05-17T15:14:46.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1094151168 bytes
2024-05-17T15:14:46.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1094757376 bytes
2024-05-17T15:14:46.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1155205120 bytes
2024-05-17T15:14:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1171890176 bytes
2024-05-17T15:14:47.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1262434304 bytes
2024-05-17T15:14:47.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1299184640 bytes
2024-05-17T15:14:47.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1331805184 bytes
2024-05-17T15:14:47.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1412582400 bytes
2024-05-17T15:14:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1527661568 bytes
2024-05-17T15:14:48.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1560450048 bytes
2024-05-17T15:14:48.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1643793408 bytes
2024-05-17T15:14:48.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1656065024 bytes
2024-05-17T15:14:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1658258432 bytes
2024-05-17T15:14:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1713371136 bytes
2024-05-17T15:14:49.241 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:14:49.240+0000 7f28b79e2640 -1 reset not still connected to 0x5625d4277ba0
2024-05-17T15:14:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1754749952 bytes
2024-05-17T15:14:49.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1877009408 bytes
2024-05-17T15:14:49.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1877654528 bytes
2024-05-17T15:14:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1894807552 bytes
2024-05-17T15:14:50.152 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1929086976 bytes
2024-05-17T15:14:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1952987136 bytes
2024-05-17T15:14:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1982561280 bytes
2024-05-17T15:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2021008384 bytes
2024-05-17T15:14:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2049977344 bytes
2024-05-17T15:14:51.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2089729024 bytes
2024-05-17T15:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2102851584 bytes
2024-05-17T15:14:51.312 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2124111872 bytes
2024-05-17T15:14:51.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2180249600 bytes
2024-05-17T15:14:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2242236416 bytes
2024-05-17T15:14:51.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2270056448 bytes
2024-05-17T15:14:52.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2328527872 bytes
2024-05-17T15:14:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2429539328 bytes
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_41
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:52.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_41
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_41 128M
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_41
2024-05-17T15:14:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:14:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:14:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:14:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:14:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_41
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_41 snapshot
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_41
2024-05-17T15:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot
2024-05-17T15:14:53.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:14:53.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_41
2024-05-17T15:14:53.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_41':
2024-05-17T15:14:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:14:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:14:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:14:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 147970658f18
2024-05-17T15:14:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.147970658f18
2024-05-17T15:14:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:14:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:14:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:14:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:14:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:14:52 2024
2024-05-17T15:14:53.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:14:52 2024
2024-05-17T15:14:53.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:14:52 2024
2024-05-17T15:14:53.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:14:53.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:14:53.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 6b91ec04-fc7b-4683-a9c9-82e9cfb0f9a6
2024-05-17T15:14:53.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:14:53.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:14:53.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_41
2024-05-17T15:14:53.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:14:53.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:14:53.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_41
2024-05-17T15:14:53.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:14:53.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:14:53.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:14:53.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:14:53.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 50919424 bytes
2024-05-17T15:14:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 60121088 bytes
2024-05-17T15:14:54.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 98101248 bytes
2024-05-17T15:14:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 132339712 bytes
2024-05-17T15:14:54.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 174301184 bytes
2024-05-17T15:14:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 184953856 bytes
2024-05-17T15:14:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 257518592 bytes
2024-05-17T15:14:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 285009920 bytes
2024-05-17T15:14:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 291618816 bytes
2024-05-17T15:14:55.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 364028928 bytes
2024-05-17T15:14:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 364663808 bytes
2024-05-17T15:14:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 415174656 bytes
2024-05-17T15:14:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 473307136 bytes
2024-05-17T15:14:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 509156352 bytes
2024-05-17T15:14:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 509801472 bytes
2024-05-17T15:14:56.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 543284224 bytes
2024-05-17T15:14:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 639803392 bytes
2024-05-17T15:14:57.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 710899712 bytes
2024-05-17T15:14:57.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 720849920 bytes
2024-05-17T15:14:57.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 768256000 bytes
2024-05-17T15:14:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 778943488 bytes
2024-05-17T15:14:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 800410624 bytes
2024-05-17T15:14:57.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 835189760 bytes
2024-05-17T15:14:57.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 859800576 bytes
2024-05-17T15:14:58.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 924730368 bytes
2024-05-17T15:14:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 972476416 bytes
2024-05-17T15:14:58.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1004857344 bytes
2024-05-17T15:14:58.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1060792320 bytes
2024-05-17T15:14:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1075243008 bytes
2024-05-17T15:14:59.246 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1131232256 bytes
2024-05-17T15:14:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1166000128 bytes
2024-05-17T15:14:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1248016384 bytes
2024-05-17T15:14:59.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1295511552 bytes
2024-05-17T15:15:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1349866496 bytes
2024-05-17T15:15:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1416525824 bytes
2024-05-17T15:15:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1466488832 bytes
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_42
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_42
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_42 128M
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_42
2024-05-17T15:15:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:15:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:15:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:15:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_42
2024-05-17T15:15:00.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:15:00.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:00.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:15:00.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:00.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_42 snapshot
2024-05-17T15:15:00.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:00.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:00.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_42
2024-05-17T15:15:00.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:00.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot
2024-05-17T15:15:01.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:15:01.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_42
2024-05-17T15:15:01.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_42':
2024-05-17T15:15:01.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14851bc63ce6
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14851bc63ce6
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:15:00 2024
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:15:00 2024
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:15:00 2024
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:15:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 4e765562-88a4-4731-be65-1c1f5d92c710
2024-05-17T15:15:01.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:15:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:15:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_42
2024-05-17T15:15:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_42
2024-05-17T15:15:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:15:01.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=9
2024-05-17T15:15:01.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:15:01.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:15:01.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 13487104 bytes
2024-05-17T15:15:02.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 84431872 bytes
2024-05-17T15:15:02.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 133539840 bytes
2024-05-17T15:15:02.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 166628352 bytes
2024-05-17T15:15:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 231140352 bytes
2024-05-17T15:15:02.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 281007104 bytes
2024-05-17T15:15:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 300811264 bytes
2024-05-17T15:15:03.215 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 303062016 bytes
2024-05-17T15:15:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 358422528 bytes
2024-05-17T15:15:03.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 385415168 bytes
2024-05-17T15:15:03.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 415072256 bytes
2024-05-17T15:15:04.026 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 477431808 bytes
2024-05-17T15:15:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 545414144 bytes
2024-05-17T15:15:04.416 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 563585024 bytes
2024-05-17T15:15:04.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 594898944 bytes
2024-05-17T15:15:04.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 620520448 bytes
2024-05-17T15:15:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 637566976 bytes
2024-05-17T15:15:05.013 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 653213696 bytes
2024-05-17T15:15:05.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 707664896 bytes
2024-05-17T15:15:05.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 782981120 bytes
2024-05-17T15:15:05.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 863906816 bytes
2024-05-17T15:15:05.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 900746240 bytes
2024-05-17T15:15:06.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 901373952 bytes
2024-05-17T15:15:06.337 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 964593664 bytes
2024-05-17T15:15:06.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 977745920 bytes
2024-05-17T15:15:06.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1059699712 bytes
2024-05-17T15:15:06.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1117860864 bytes
2024-05-17T15:15:07.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1157950464 bytes
2024-05-17T15:15:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1201268736 bytes
2024-05-17T15:15:07.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1224414208 bytes
2024-05-17T15:15:07.700 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1255537664 bytes
2024-05-17T15:15:07.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1300466688 bytes
2024-05-17T15:15:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1341568000 bytes
2024-05-17T15:15:08.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1424812032 bytes
2024-05-17T15:15:08.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1469075456 bytes
2024-05-17T15:15:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1469709312 bytes
2024-05-17T15:15:08.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1496843264 bytes
2024-05-17T15:15:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1560692736 bytes
2024-05-17T15:15:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1647348736 bytes
2024-05-17T15:15:09.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1667238912 bytes
2024-05-17T15:15:09.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1693147136 bytes
2024-05-17T15:15:09.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1714699264 bytes
2024-05-17T15:15:10.044 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9458065525836798) > weight (0.25)
2024-05-17T15:15:10.044 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.525160359946605) > weight (0.25)
2024-05-17T15:15:10.044 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2
2024-05-17T15:15:10.044 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15
2024-05-17T15:15:10.045 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons
2024-05-17T15:15:10.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1727653888 bytes
2024-05-17T15:15:10.046 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi171.stderr:2024-05-17T15:15:10.044+0000 7f1c44553640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2  (PID: 32463) UID: 1000
2024-05-17T15:15:10.209 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1751791616 bytes
2024-05-17T15:15:10.352 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1766397952 bytes
2024-05-17T15:15:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1783615488 bytes
2024-05-17T15:15:10.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:15:10.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:15:10.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:15:10.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:10.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_43
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_43
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_43 128M
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_43
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_43
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_43 snapshot
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_43
2024-05-17T15:15:10.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:10.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot
2024-05-17T15:15:11.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:15:11.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_43
2024-05-17T15:15:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_43':
2024-05-17T15:15:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1490be6eac17
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1490be6eac17
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:15:10 2024
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:15:10 2024
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:15:10 2024
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:15:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: fde9710b-b854-4036-8fd7-b96cfafd23f3
2024-05-17T15:15:11.709 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:15:11.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:15:11.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_43
2024-05-17T15:15:11.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_43
2024-05-17T15:15:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:15:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=12
2024-05-17T15:15:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:15:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:15:11.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 43833344 bytes
2024-05-17T15:15:12.046 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2
2024-05-17T15:15:12.046 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:15:12.047 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2
2024-05-17T15:15:12.047 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon
2024-05-17T15:15:12.047 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2
2024-05-17T15:15:12.049 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started
2024-05-17T15:15:12.049 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing
2024-05-17T15:15:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 90510336 bytes
2024-05-17T15:15:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 141992960 bytes
2024-05-17T15:15:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 186529792 bytes
2024-05-17T15:15:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 269520896 bytes
2024-05-17T15:15:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 327239680 bytes
2024-05-17T15:15:13.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 375010304 bytes
2024-05-17T15:15:13.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 400983040 bytes
2024-05-17T15:15:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 445193216 bytes
2024-05-17T15:15:13.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 488658944 bytes
2024-05-17T15:15:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 515075072 bytes
2024-05-17T15:15:14.231 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 551377920 bytes
2024-05-17T15:15:14.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 572682240 bytes
2024-05-17T15:15:14.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 580192256 bytes
2024-05-17T15:15:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 632512512 bytes
2024-05-17T15:15:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 688161792 bytes
2024-05-17T15:15:15.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 708558848 bytes
2024-05-17T15:15:15.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 777567232 bytes
2024-05-17T15:15:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 814585856 bytes
2024-05-17T15:15:15.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 889975808 bytes
2024-05-17T15:15:15.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 914314240 bytes
2024-05-17T15:15:16.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 964923392 bytes
2024-05-17T15:15:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1020193792 bytes
2024-05-17T15:15:16.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1033987072 bytes
2024-05-17T15:15:16.681 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1078579200 bytes
2024-05-17T15:15:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1129200640 bytes
2024-05-17T15:15:17.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1213488128 bytes
2024-05-17T15:15:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1259811840 bytes
2024-05-17T15:15:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1292197888 bytes
2024-05-17T15:15:17.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1310974976 bytes
2024-05-17T15:15:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1333642240 bytes
2024-05-17T15:15:18.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1384248320 bytes
2024-05-17T15:15:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1428402176 bytes
2024-05-17T15:15:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1511019520 bytes
2024-05-17T15:15:18.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1531492352 bytes
2024-05-17T15:15:18.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1559240704 bytes
2024-05-17T15:15:19.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1563978752 bytes
2024-05-17T15:15:19.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1598593024 bytes
2024-05-17T15:15:19.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1629392896 bytes
2024-05-17T15:15:19.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1720164352 bytes
2024-05-17T15:15:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1763584000 bytes
2024-05-17T15:15:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1815145472 bytes
2024-05-17T15:15:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1854101504 bytes
2024-05-17T15:15:20.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1909340160 bytes
2024-05-17T15:15:20.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 2030287872 bytes
2024-05-17T15:15:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2056727552 bytes
2024-05-17T15:15:21.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2077293568 bytes
2024-05-17T15:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2152295424 bytes
2024-05-17T15:15:21.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2181425152 bytes
2024-05-17T15:15:21.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2231079936 bytes
2024-05-17T15:15:21.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2256175104 bytes
2024-05-17T15:15:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2326968320 bytes
2024-05-17T15:15:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2372716544 bytes
2024-05-17T15:15:22.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2391064576 bytes
2024-05-17T15:15:22.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2406109184 bytes
2024-05-17T15:15:22.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2431374336 bytes
2024-05-17T15:15:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2432004096 bytes
2024-05-17T15:15:23.218 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2501428224 bytes
2024-05-17T15:15:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2592447488 bytes
2024-05-17T15:15:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:15:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:15:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:15:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_44
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_44
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_44 128M
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_44
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_44
2024-05-17T15:15:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:15:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:15:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_44 snapshot
2024-05-17T15:15:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:23.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:23.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_44
2024-05-17T15:15:23.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:23.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot
2024-05-17T15:15:24.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:15:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_44
2024-05-17T15:15:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_44':
2024-05-17T15:15:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 149e69c394c1
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.149e69c394c1
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:15:23 2024
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:15:23 2024
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:15:23 2024
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:15:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ea7b1da0-05fb-45d7-867b-e5e9cb6b58b4
2024-05-17T15:15:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:15:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:15:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_44
2024-05-17T15:15:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_44
2024-05-17T15:15:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:15:24.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:15:24.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:15:24.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:15:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 33441792 bytes
2024-05-17T15:15:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 53639168 bytes
2024-05-17T15:15:25.371 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 77371392 bytes
2024-05-17T15:15:25.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 133496832 bytes
2024-05-17T15:15:25.799 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 193357824 bytes
2024-05-17T15:15:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 240088064 bytes
2024-05-17T15:15:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 298232832 bytes
2024-05-17T15:15:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 416224256 bytes
2024-05-17T15:15:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 435821568 bytes
2024-05-17T15:15:26.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 465540096 bytes
2024-05-17T15:15:27.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 505869312 bytes
2024-05-17T15:15:27.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 509874176 bytes
2024-05-17T15:15:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 537304064 bytes
2024-05-17T15:15:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 624944128 bytes
2024-05-17T15:15:27.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 692882432 bytes
2024-05-17T15:15:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 751259648 bytes
2024-05-17T15:15:28.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 759131136 bytes
2024-05-17T15:15:28.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 781122560 bytes
2024-05-17T15:15:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 829104128 bytes
2024-05-17T15:15:28.849 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 880199680 bytes
2024-05-17T15:15:29.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 903052288 bytes
2024-05-17T15:15:29.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 939696128 bytes
2024-05-17T15:15:29.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 973252608 bytes
2024-05-17T15:15:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 998218752 bytes
2024-05-17T15:15:29.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1022802944 bytes
2024-05-17T15:15:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1046136832 bytes
2024-05-17T15:15:29.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1046718464 bytes
2024-05-17T15:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1134820352 bytes
2024-05-17T15:15:30.384 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1150298112 bytes
2024-05-17T15:15:30.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1215051776 bytes
2024-05-17T15:15:30.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1228772352 bytes
2024-05-17T15:15:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1277090816 bytes
2024-05-17T15:15:31.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1301440512 bytes
2024-05-17T15:15:31.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1352173568 bytes
2024-05-17T15:15:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1434701824 bytes
2024-05-17T15:15:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1488115712 bytes
2024-05-17T15:15:31.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1502756864 bytes
2024-05-17T15:15:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1545562112 bytes
2024-05-17T15:15:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1578895360 bytes
2024-05-17T15:15:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1588158464 bytes
2024-05-17T15:15:32.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1626181632 bytes
2024-05-17T15:15:32.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1672425472 bytes
2024-05-17T15:15:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1744638976 bytes
2024-05-17T15:15:33.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1800554496 bytes
2024-05-17T15:15:33.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1851329536 bytes
2024-05-17T15:15:33.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1874948096 bytes
2024-05-17T15:15:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1922807808 bytes
2024-05-17T15:15:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1973923840 bytes
2024-05-17T15:15:34.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2031848448 bytes
2024-05-17T15:15:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2085354496 bytes
2024-05-17T15:15:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2109003776 bytes
2024-05-17T15:15:34.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2127650816 bytes
2024-05-17T15:15:35.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2173835264 bytes
2024-05-17T15:15:35.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2209155072 bytes
2024-05-17T15:15:35.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2209772544 bytes
2024-05-17T15:15:35.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2254618624 bytes
2024-05-17T15:15:35.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2273789952 bytes
2024-05-17T15:15:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2342806528 bytes
2024-05-17T15:15:36.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2431538176 bytes
2024-05-17T15:15:36.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2486294528 bytes
2024-05-17T15:15:36.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2488192000 bytes
2024-05-17T15:15:36.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2536991744 bytes
2024-05-17T15:15:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2623033344 bytes
2024-05-17T15:15:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2753809408 bytes
2024-05-17T15:15:37.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2776444928 bytes
2024-05-17T15:15:37.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:15:37.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_45
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_45
2024-05-17T15:15:37.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_45 128M
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_45
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:15:37.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:37.844 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_45
2024-05-17T15:15:37.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_45 snapshot
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_45
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:37.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot
2024-05-17T15:15:38.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:15:38.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_45
2024-05-17T15:15:38.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_45':
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14aac4472339
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14aac4472339
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:15:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:15:37 2024
2024-05-17T15:15:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:15:37 2024
2024-05-17T15:15:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:15:37 2024
2024-05-17T15:15:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:15:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:15:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: e4fac0ec-ba6a-4337-b659-b2a86fce1362
2024-05-17T15:15:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:15:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:15:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_45
2024-05-17T15:15:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_45
2024-05-17T15:15:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:15:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=5
2024-05-17T15:15:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:15:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:15:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 62685184 bytes
2024-05-17T15:15:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 109498368 bytes
2024-05-17T15:15:39.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 141144064 bytes
2024-05-17T15:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 146038784 bytes
2024-05-17T15:15:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 209730560 bytes
2024-05-17T15:15:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 243693568 bytes
2024-05-17T15:15:40.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 261392384 bytes
2024-05-17T15:15:40.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 293766144 bytes
2024-05-17T15:15:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 356920320 bytes
2024-05-17T15:15:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 357532672 bytes
2024-05-17T15:15:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 388167680 bytes
2024-05-17T15:15:41.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 447393792 bytes
2024-05-17T15:15:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 487964672 bytes
2024-05-17T15:15:41.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 512911360 bytes
2024-05-17T15:15:41.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 555552768 bytes
2024-05-17T15:15:41.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 603720704 bytes
2024-05-17T15:15:41.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 633207808 bytes
2024-05-17T15:15:42.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 649332736 bytes
2024-05-17T15:15:42.204 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 656940032 bytes
2024-05-17T15:15:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 745837568 bytes
2024-05-17T15:15:42.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 751047680 bytes
2024-05-17T15:15:42.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 767170560 bytes
2024-05-17T15:15:42.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 820793344 bytes
2024-05-17T15:15:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 829578240 bytes
2024-05-17T15:15:43.325 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 901553152 bytes
2024-05-17T15:15:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 918584320 bytes
2024-05-17T15:15:43.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:15:43.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_46
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_46
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_46 128M
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_46
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_46
2024-05-17T15:15:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:15:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:15:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_46 snapshot
2024-05-17T15:15:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_46
2024-05-17T15:15:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot
2024-05-17T15:15:44.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:15:44.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_46
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_46':
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14b64b2fba6f
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14b64b2fba6f
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:15:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:15:43 2024
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:15:43 2024
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:15:43 2024
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 433ce424-b82f-47dd-a3e8-e92df81bb4c4
2024-05-17T15:15:44.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:15:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:15:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_46
2024-05-17T15:15:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_46
2024-05-17T15:15:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:15:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:15:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:15:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:15:44.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 16644096 bytes
2024-05-17T15:15:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 70149120 bytes
2024-05-17T15:15:45.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 116517888 bytes
2024-05-17T15:15:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 196470784 bytes
2024-05-17T15:15:45.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 216608768 bytes
2024-05-17T15:15:45.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 296132608 bytes
2024-05-17T15:15:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 352616448 bytes
2024-05-17T15:15:46.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 400801792 bytes
2024-05-17T15:15:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 436578304 bytes
2024-05-17T15:15:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 488000512 bytes
2024-05-17T15:15:46.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 558008320 bytes
2024-05-17T15:15:47.082 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 582874112 bytes
2024-05-17T15:15:47.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 609893376 bytes
2024-05-17T15:15:47.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 648449024 bytes
2024-05-17T15:15:47.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 693023744 bytes
2024-05-17T15:15:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 753198080 bytes
2024-05-17T15:15:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 778450944 bytes
2024-05-17T15:15:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 779899904 bytes
2024-05-17T15:15:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 830527488 bytes
2024-05-17T15:15:48.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 891249664 bytes
2024-05-17T15:15:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 926759936 bytes
2024-05-17T15:15:48.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 948869120 bytes
2024-05-17T15:15:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 982834176 bytes
2024-05-17T15:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1007776768 bytes
2024-05-17T15:15:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1021675520 bytes
2024-05-17T15:15:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1052816384 bytes
2024-05-17T15:15:49.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1075042304 bytes
2024-05-17T15:15:49.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1114132480 bytes
2024-05-17T15:15:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1133648896 bytes
2024-05-17T15:15:50.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1149765632 bytes
2024-05-17T15:15:50.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1197140992 bytes
2024-05-17T15:15:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1220517888 bytes
2024-05-17T15:15:50.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1254125568 bytes
2024-05-17T15:15:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1279057920 bytes
2024-05-17T15:15:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1297640448 bytes
2024-05-17T15:15:51.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1346860032 bytes
2024-05-17T15:15:51.392 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1385893888 bytes
2024-05-17T15:15:51.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1431180288 bytes
2024-05-17T15:15:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1477221376 bytes
2024-05-17T15:15:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1519519744 bytes
2024-05-17T15:15:52.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1546614784 bytes
2024-05-17T15:15:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1576887296 bytes
2024-05-17T15:15:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1625548800 bytes
2024-05-17T15:15:52.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1650639872 bytes
2024-05-17T15:15:52.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1720009728 bytes
2024-05-17T15:15:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1733092352 bytes
2024-05-17T15:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 1765638144 bytes
2024-05-17T15:15:53.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 1822631936 bytes
2024-05-17T15:15:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 1843529728 bytes
2024-05-17T15:15:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 1884613632 bytes
2024-05-17T15:15:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 1891667968 bytes
2024-05-17T15:15:54.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 1928116224 bytes
2024-05-17T15:15:54.329 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 1983911936 bytes
2024-05-17T15:15:54.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2067160064 bytes
2024-05-17T15:15:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2136943616 bytes
2024-05-17T15:15:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2177334272 bytes
2024-05-17T15:15:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2202030080 bytes
2024-05-17T15:15:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:15:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:15:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:15:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:15:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_47
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_47
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_47 128M
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_47
2024-05-17T15:15:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:15:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:15:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:15:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:15:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_47
2024-05-17T15:15:55.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_47 snapshot
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_47
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:15:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot
2024-05-17T15:15:56.048 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8418514020145799) > weight (0.25)
2024-05-17T15:15:56.048 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9958169536883231) > weight (0.25)
2024-05-17T15:15:56.048 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T15:15:56.049 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T15:15:56.049 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons
2024-05-17T15:15:56.050 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T15:15:56.050+0000 7fa534360640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 41019) UID: 1000
2024-05-17T15:15:56.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:15:56.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_47
2024-05-17T15:15:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_47':
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14c27f9029c2
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14c27f9029c2
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:15:55 2024
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:15:55 2024
2024-05-17T15:15:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:15:55 2024
2024-05-17T15:15:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:15:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:15:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 6b202532-2b8e-4b7c-8954-7bffc20d215f
2024-05-17T15:15:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:15:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:15:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_47
2024-05-17T15:15:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:15:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:15:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_47
2024-05-17T15:15:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:15:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=14
2024-05-17T15:15:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:15:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:15:56.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 29735936 bytes
2024-05-17T15:15:57.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 64921600 bytes
2024-05-17T15:15:57.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 87244800 bytes
2024-05-17T15:15:57.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 109131776 bytes
2024-05-17T15:15:57.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 223300608 bytes
2024-05-17T15:15:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 259684352 bytes
2024-05-17T15:15:58.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 291656704 bytes
2024-05-17T15:15:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 318615552 bytes
2024-05-17T15:15:58.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 319229952 bytes
2024-05-17T15:15:58.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 417113088 bytes
2024-05-17T15:15:58.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 465409024 bytes
2024-05-17T15:15:59.049 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T15:15:59.050 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:15:59.050 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T15:15:59.050 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T15:15:59.050 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T15:15:59.053 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T15:15:59.053 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing
2024-05-17T15:15:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 534920192 bytes
2024-05-17T15:15:59.363 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 565961728 bytes
2024-05-17T15:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 575337472 bytes
2024-05-17T15:15:59.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 626649088 bytes
2024-05-17T15:16:00.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 704358400 bytes
2024-05-17T15:16:00.191 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 729951232 bytes
2024-05-17T15:16:00.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 821109760 bytes
2024-05-17T15:16:00.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 926348288 bytes
2024-05-17T15:16:01.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 1002895360 bytes
2024-05-17T15:16:01.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1063990272 bytes
2024-05-17T15:16:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1094436864 bytes
2024-05-17T15:16:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1100611584 bytes
2024-05-17T15:16:01.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1114831872 bytes
2024-05-17T15:16:01.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1124096000 bytes
2024-05-17T15:16:02.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1169857536 bytes
2024-05-17T15:16:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1199755264 bytes
2024-05-17T15:16:02.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1269942272 bytes
2024-05-17T15:16:02.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1291976704 bytes
2024-05-17T15:16:02.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1308444672 bytes
2024-05-17T15:16:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1378356224 bytes
2024-05-17T15:16:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1404328960 bytes
2024-05-17T15:16:03.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1431769088 bytes
2024-05-17T15:16:03.504 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1452302336 bytes
2024-05-17T15:16:03.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1498974208 bytes
2024-05-17T15:16:03.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1511493632 bytes
2024-05-17T15:16:04.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1561862144 bytes
2024-05-17T15:16:04.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1615891456 bytes
2024-05-17T15:16:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1642306560 bytes
2024-05-17T15:16:04.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1642918912 bytes
2024-05-17T15:16:04.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1721249792 bytes
2024-05-17T15:16:05.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1768806400 bytes
2024-05-17T15:16:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1848393728 bytes
2024-05-17T15:16:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1896456192 bytes
2024-05-17T15:16:05.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1932793856 bytes
2024-05-17T15:16:05.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2043225088 bytes
2024-05-17T15:16:06.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2120775680 bytes
2024-05-17T15:16:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2272514048 bytes
2024-05-17T15:16:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2377684992 bytes
2024-05-17T15:16:07.209 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2387097600 bytes
2024-05-17T15:16:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2436207616 bytes
2024-05-17T15:16:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2486746112 bytes
2024-05-17T15:16:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2567187456 bytes
2024-05-17T15:16:08.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2663246848 bytes
2024-05-17T15:16:08.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2705974272 bytes
2024-05-17T15:16:08.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2708893696 bytes
2024-05-17T15:16:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2749312000 bytes
2024-05-17T15:16:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2813607936 bytes
2024-05-17T15:16:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2856837120 bytes
2024-05-17T15:16:09.431 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2927407104 bytes
2024-05-17T15:16:09.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2987070464 bytes
2024-05-17T15:16:09.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 3023202304 bytes
2024-05-17T15:16:10.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 3044285440 bytes
2024-05-17T15:16:10.380 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 3206222848 bytes
2024-05-17T15:16:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 3254563840 bytes
2024-05-17T15:16:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:16:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:16:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:16:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_48
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_48
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_48 128M
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_48
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_48
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_48 snapshot
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_48
2024-05-17T15:16:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:16:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot
2024-05-17T15:16:11.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:16:11.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_48
2024-05-17T15:16:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_48':
2024-05-17T15:16:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:16:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:16:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14d64c793cb7
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14d64c793cb7
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:16:10 2024
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:16:10 2024
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:16:10 2024
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 0577c8a6-efc8-4ea8-8ff1-85e30dfc35b7
2024-05-17T15:16:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:16:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:16:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_48
2024-05-17T15:16:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_48
2024-05-17T15:16:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:16:11.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=13
2024-05-17T15:16:11.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:16:11.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:16:12.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 76877824 bytes
2024-05-17T15:16:12.443 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 183957504 bytes
2024-05-17T15:16:12.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 207826944 bytes
2024-05-17T15:16:12.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 258677760 bytes
2024-05-17T15:16:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 271693824 bytes
2024-05-17T15:16:13.051 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.848870082729985) > weight (0.25)
2024-05-17T15:16:13.051 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9195355984101036) > weight (0.25)
2024-05-17T15:16:13.051 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9314302997579885) > weight (0.25)
2024-05-17T15:16:13.051 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7936651074111228) > weight (0.25)
2024-05-17T15:16:13.051 INFO:tasks.rbd_mirror_thrash:waiting for 111 secs before thrashing
2024-05-17T15:16:13.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 314838016 bytes
2024-05-17T15:16:13.269 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 322788352 bytes
2024-05-17T15:16:13.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 371062784 bytes
2024-05-17T15:16:13.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 396326912 bytes
2024-05-17T15:16:13.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 491533312 bytes
2024-05-17T15:16:14.097 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 516273152 bytes
2024-05-17T15:16:14.348 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 596434944 bytes
2024-05-17T15:16:14.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 623058944 bytes
2024-05-17T15:16:14.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 630787072 bytes
2024-05-17T15:16:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 637126656 bytes
2024-05-17T15:16:14.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 637735936 bytes
2024-05-17T15:16:15.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 647035904 bytes
2024-05-17T15:16:15.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 671546368 bytes
2024-05-17T15:16:15.417 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 715189248 bytes
2024-05-17T15:16:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 756289536 bytes
2024-05-17T15:16:15.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 865846272 bytes
2024-05-17T15:16:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 913850368 bytes
2024-05-17T15:16:16.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 947223552 bytes
2024-05-17T15:16:16.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1013007360 bytes
2024-05-17T15:16:16.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1061121024 bytes
2024-05-17T15:16:17.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1125427200 bytes
2024-05-17T15:16:17.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1192147968 bytes
2024-05-17T15:16:17.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1278313472 bytes
2024-05-17T15:16:17.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1383989248 bytes
2024-05-17T15:16:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1384607744 bytes
2024-05-17T15:16:18.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1424074752 bytes
2024-05-17T15:16:18.322 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1433390080 bytes
2024-05-17T15:16:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1463627776 bytes
2024-05-17T15:16:18.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1522649088 bytes
2024-05-17T15:16:18.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1546540032 bytes
2024-05-17T15:16:19.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1611185152 bytes
2024-05-17T15:16:19.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1667382272 bytes
2024-05-17T15:16:19.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1677904896 bytes
2024-05-17T15:16:19.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1711225856 bytes
2024-05-17T15:16:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1745475584 bytes
2024-05-17T15:16:19.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1766618112 bytes
2024-05-17T15:16:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1842377728 bytes
2024-05-17T15:16:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1867891712 bytes
2024-05-17T15:16:20.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1924193280 bytes
2024-05-17T15:16:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1946545152 bytes
2024-05-17T15:16:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 1986899968 bytes
2024-05-17T15:16:21.133 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2012975104 bytes
2024-05-17T15:16:21.363 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T15:16:21.362+0000 7fc89be28640 -1 bdev(0x5588ebcece00 /var/lib/ceph/osd/cluster2-0/block) _aio_thread got r=-28 ((28) No space left on device)
2024-05-17T15:16:21.364 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:./src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fc89be28640 time 2024-05-17T15:16:21.365709+0000
2024-05-17T15:16:21.364 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:./src/blk/kernel/KernelDevice.cc: 666: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!")
2024-05-17T15:16:21.364 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xc6) [0x5588e8312bd5]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T15:16:21.362+0000 7fc89be28640 -1 ./src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fc89be28640 time 2024-05-17T15:16:21.365709+0000
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:./src/blk/kernel/KernelDevice.cc: 666: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!")
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xc6) [0x5588e8312bd5]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.365 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.366 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.366 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.366 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:*** Caught signal (Aborted) **
2024-05-17T15:16:21.366 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: in thread 7fc89be28640 thread_name:bstore_aio
2024-05-17T15:16:21.366 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.366 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fc8a9415520]
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: pthread_kill()
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: raise()
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: abort()
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x17b) [0x5588e8312c8a]
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 6: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 7: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 8: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:2024-05-17T15:16:21.362+0000 7fc89be28640 -1 *** Caught signal (Aborted) **
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: in thread 7fc89be28640 thread_name:bstore_aio
2024-05-17T15:16:21.367 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fc8a9415520]
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: pthread_kill()
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: raise()
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: abort()
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x17b) [0x5588e8312c8a]
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 6: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 7: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 8: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:16:21.368 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:   -14> 2024-05-17T15:16:21.362+0000 7fc89be28640 -1 bdev(0x5588ebcece00 /var/lib/ceph/osd/cluster2-0/block) _aio_thread got r=-28 ((28) No space left on device)
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:    -1> 2024-05-17T15:16:21.362+0000 7fc89be28640 -1 ./src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fc89be28640 time 2024-05-17T15:16:21.365709+0000
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:./src/blk/kernel/KernelDevice.cc: 666: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!")
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xc6) [0x5588e8312bd5]
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.379 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:     0> 2024-05-17T15:16:21.362+0000 7fc89be28640 -1 *** Caught signal (Aborted) **
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: in thread 7fc89be28640 thread_name:bstore_aio
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fc8a9415520]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: pthread_kill()
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: raise()
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: abort()
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x17b) [0x5588e8312c8a]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 6: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 7: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.380 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 8: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.381 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.381 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:16:21.381 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:   -77> 2024-05-17T15:16:21.362+0000 7fc89be28640 -1 bdev(0x5588ebcece00 /var/lib/ceph/osd/cluster2-0/block) _aio_thread got r=-28 ((28) No space left on device)
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:   -64> 2024-05-17T15:16:21.362+0000 7fc89be28640 -1 ./src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fc89be28640 time 2024-05-17T15:16:21.365709+0000
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:./src/blk/kernel/KernelDevice.cc: 666: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!")
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xc6) [0x5588e8312bd5]
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.392 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:   -63> 2024-05-17T15:16:21.362+0000 7fc89be28640 -1 *** Caught signal (Aborted) **
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: in thread 7fc89be28640 thread_name:bstore_aio
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fc8a9415520]
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 2: pthread_kill()
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 3: raise()
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 4: abort()
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x17b) [0x5588e8312c8a]
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 6: (KernelDevice::_aio_thread()+0x9e2) [0x5588e8ccaab2]
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 7: ceph-osd(+0xd92631) [0x5588e8cae631]
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 8: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc8a9467b43]
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc8a94f9a00]
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:16:21.393 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:
2024-05-17T15:16:21.590 INFO:tasks.ceph.cluster2.osd.0.smithi171.stderr:daemon-helper: command failed with exit status 5
2024-05-17T15:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2089809920 bytes
2024-05-17T15:16:23.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2120205312 bytes
2024-05-17T15:16:23.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2170277888 bytes
2024-05-17T15:16:24.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2220781568 bytes
2024-05-17T15:16:24.152 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2256726016 bytes
2024-05-17T15:16:24.271 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2298329088 bytes
2024-05-17T15:16:24.340 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s
2024-05-17T15:16:24.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2414061568 bytes
2024-05-17T15:16:24.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2438748160 bytes
2024-05-17T15:16:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2512881664 bytes
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_49
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_49
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_49 128M
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_49
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:16:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:16:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:16:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_49
2024-05-17T15:16:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:16:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:16:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_49 snapshot
2024-05-17T15:16:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_49
2024-05-17T15:16:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:16:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot
2024-05-17T15:16:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:16:25.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_49
2024-05-17T15:16:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_49':
2024-05-17T15:16:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14e26e54ccbd
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14e26e54ccbd
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:16:24 2024
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:16:24 2024
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:16:24 2024
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:16:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 1dc41e89-a5f9-4f8c-93d2-dd0336be3c3b
2024-05-17T15:16:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:16:25.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:16:25.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_49
2024-05-17T15:16:25.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:25.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:25.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_49
2024-05-17T15:16:25.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:16:25.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=7
2024-05-17T15:16:25.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:16:25.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:16:25.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 85892096 bytes
2024-05-17T15:16:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 156365824 bytes
2024-05-17T15:16:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 192923648 bytes
2024-05-17T15:16:26.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 209273856 bytes
2024-05-17T15:16:26.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 266783744 bytes
2024-05-17T15:16:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 277248000 bytes
2024-05-17T15:16:26.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 383971328 bytes
2024-05-17T15:16:27.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 467764224 bytes
2024-05-17T15:16:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 490365952 bytes
2024-05-17T15:16:27.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 546383872 bytes
2024-05-17T15:16:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 580408320 bytes
2024-05-17T15:16:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 599052288 bytes
2024-05-17T15:16:27.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 692980736 bytes
2024-05-17T15:16:28.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 727399424 bytes
2024-05-17T15:16:28.213 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 769858560 bytes
2024-05-17T15:16:28.392 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 854089728 bytes
2024-05-17T15:16:28.530 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 878774272 bytes
2024-05-17T15:16:28.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 895392768 bytes
2024-05-17T15:16:28.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 945598464 bytes
2024-05-17T15:16:29.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 1016481792 bytes
2024-05-17T15:16:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 1091833856 bytes
2024-05-17T15:16:29.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 1123837952 bytes
2024-05-17T15:16:29.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1185045504 bytes
2024-05-17T15:16:29.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1238423552 bytes
2024-05-17T15:16:29.744 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~5s
2024-05-17T15:16:29.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1306929152 bytes
2024-05-17T15:16:30.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1353464832 bytes
2024-05-17T15:16:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1451242496 bytes
2024-05-17T15:16:30.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1518559232 bytes
2024-05-17T15:16:30.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1587069952 bytes
2024-05-17T15:16:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1595433984 bytes
2024-05-17T15:16:30.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1622482944 bytes
2024-05-17T15:16:31.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1668426752 bytes
2024-05-17T15:16:31.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1676089344 bytes
2024-05-17T15:16:31.277 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1696833536 bytes
2024-05-17T15:16:31.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1765319680 bytes
2024-05-17T15:16:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1773233152 bytes
2024-05-17T15:16:31.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1822998528 bytes
2024-05-17T15:16:31.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1832574976 bytes
2024-05-17T15:16:31.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1867722752 bytes
2024-05-17T15:16:32.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1946205184 bytes
2024-05-17T15:16:32.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1946823680 bytes
2024-05-17T15:16:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 2019217408 bytes
2024-05-17T15:16:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 2038377472 bytes
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_50
2024-05-17T15:16:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_50
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n snapshot ']'
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_image cluster2 mirror image_50 128M
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:32.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_50
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local size=128
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n 128M ']'
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ size=128M
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ shift
2024-05-17T15:16:32.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_50
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ is_pool_mirror_mode_image mirror
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' image = image ']'
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ enable_mirror cluster2 mirror image_50 snapshot
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:32.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_50
2024-05-17T15:16:32.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local mode=snapshot
2024-05-17T15:16:32.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot
2024-05-17T15:16:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Mirroring enabled
2024-05-17T15:16:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster=cluster2 info mirror/image_50
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_50':
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 1
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 14ec2daf7e50
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.14ec2daf7e50
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:16:32 2024
2024-05-17T15:16:33.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:16:32 2024
2024-05-17T15:16:33.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:16:32 2024
2024-05-17T15:16:33.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:16:33.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:16:33.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: e2bc7f70-c3f9-4e96-b32c-217bd03cc73e
2024-05-17T15:16:33.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: true
2024-05-17T15:16:33.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' -n '' ']'
2024-05-17T15:16:33.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ stress_write_image cluster2 mirror image_50
2024-05-17T15:16:33.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:33.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:33.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_50
2024-05-17T15:16:33.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}'
2024-05-17T15:16:33.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local duration=11
2024-05-17T15:16:33.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set +e
2024-05-17T15:16:33.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20
2024-05-17T15:16:33.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:100 IOs, 51165184 bytes
2024-05-17T15:16:34.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:200 IOs, 134180864 bytes
2024-05-17T15:16:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:300 IOs, 140265472 bytes
2024-05-17T15:16:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:400 IOs, 207727616 bytes
2024-05-17T15:16:34.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:500 IOs, 285394944 bytes
2024-05-17T15:16:34.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:600 IOs, 286469120 bytes
2024-05-17T15:16:34.730 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:700 IOs, 344649728 bytes
2024-05-17T15:16:34.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:800 IOs, 425605120 bytes
2024-05-17T15:16:35.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:900 IOs, 426224640 bytes
2024-05-17T15:16:35.148 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~11s
2024-05-17T15:16:35.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1000 IOs, 449285120 bytes
2024-05-17T15:16:35.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1100 IOs, 467386368 bytes
2024-05-17T15:16:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1200 IOs, 483011584 bytes
2024-05-17T15:16:35.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1300 IOs, 526645248 bytes
2024-05-17T15:16:35.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1400 IOs, 637509632 bytes
2024-05-17T15:16:35.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1500 IOs, 649326592 bytes
2024-05-17T15:16:36.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1600 IOs, 697435136 bytes
2024-05-17T15:16:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1700 IOs, 702452736 bytes
2024-05-17T15:16:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1800 IOs, 759159808 bytes
2024-05-17T15:16:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:1900 IOs, 759799808 bytes
2024-05-17T15:16:36.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2000 IOs, 825240576 bytes
2024-05-17T15:16:36.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2100 IOs, 937309184 bytes
2024-05-17T15:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2200 IOs, 962064384 bytes
2024-05-17T15:16:37.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2300 IOs, 1034553344 bytes
2024-05-17T15:16:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2400 IOs, 1071124480 bytes
2024-05-17T15:16:37.359 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2500 IOs, 1115918336 bytes
2024-05-17T15:16:37.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2600 IOs, 1155127296 bytes
2024-05-17T15:16:37.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2700 IOs, 1160138752 bytes
2024-05-17T15:16:37.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2800 IOs, 1160753152 bytes
2024-05-17T15:16:37.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:2900 IOs, 1220201472 bytes
2024-05-17T15:16:38.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3000 IOs, 1306148864 bytes
2024-05-17T15:16:38.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3100 IOs, 1306770432 bytes
2024-05-17T15:16:38.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3200 IOs, 1337783296 bytes
2024-05-17T15:16:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3300 IOs, 1399171072 bytes
2024-05-17T15:16:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3400 IOs, 1415135232 bytes
2024-05-17T15:16:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3500 IOs, 1415746560 bytes
2024-05-17T15:16:38.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3600 IOs, 1452800000 bytes
2024-05-17T15:16:38.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3700 IOs, 1485117440 bytes
2024-05-17T15:16:39.032 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3800 IOs, 1567585280 bytes
2024-05-17T15:16:39.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:3900 IOs, 1675265024 bytes
2024-05-17T15:16:39.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4000 IOs, 1749243904 bytes
2024-05-17T15:16:39.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4100 IOs, 1809444864 bytes
2024-05-17T15:16:39.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4200 IOs, 1858458624 bytes
2024-05-17T15:16:40.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4300 IOs, 1905566720 bytes
2024-05-17T15:16:40.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4400 IOs, 1934296064 bytes
2024-05-17T15:16:40.336 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4500 IOs, 1986084864 bytes
2024-05-17T15:16:40.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4600 IOs, 2052036608 bytes
2024-05-17T15:16:40.552 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~16s
2024-05-17T15:16:40.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4700 IOs, 2069190656 bytes
2024-05-17T15:16:40.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4800 IOs, 2157779968 bytes
2024-05-17T15:16:41.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:4900 IOs, 2208119808 bytes
2024-05-17T15:16:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5000 IOs, 2265393152 bytes
2024-05-17T15:16:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5100 IOs, 2315120640 bytes
2024-05-17T15:16:41.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5200 IOs, 2363629568 bytes
2024-05-17T15:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5300 IOs, 2397192192 bytes
2024-05-17T15:16:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5400 IOs, 2453170176 bytes
2024-05-17T15:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5500 IOs, 2524712960 bytes
2024-05-17T15:16:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5600 IOs, 2575884288 bytes
2024-05-17T15:16:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5700 IOs, 2618860544 bytes
2024-05-17T15:16:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5800 IOs, 2680357888 bytes
2024-05-17T15:16:42.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:5900 IOs, 2733724672 bytes
2024-05-17T15:16:42.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6000 IOs, 2775177216 bytes
2024-05-17T15:16:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6100 IOs, 2840180736 bytes
2024-05-17T15:16:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6200 IOs, 2868182016 bytes
2024-05-17T15:16:43.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6300 IOs, 2873948160 bytes
2024-05-17T15:16:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6400 IOs, 2903936000 bytes
2024-05-17T15:16:43.275 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6500 IOs, 2920977408 bytes
2024-05-17T15:16:43.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6600 IOs, 2921607168 bytes
2024-05-17T15:16:43.503 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6700 IOs, 2977878016 bytes
2024-05-17T15:16:43.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6800 IOs, 3016783872 bytes
2024-05-17T15:16:43.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:6900 IOs, 3089491968 bytes
2024-05-17T15:16:43.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7000 IOs, 3111524352 bytes
2024-05-17T15:16:44.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7100 IOs, 3118931968 bytes
2024-05-17T15:16:44.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7200 IOs, 3187251200 bytes
2024-05-17T15:16:44.272 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7300 IOs, 3189400576 bytes
2024-05-17T15:16:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7400 IOs, 3212931072 bytes
2024-05-17T15:16:44.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7500 IOs, 3242577920 bytes
2024-05-17T15:16:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:7600 IOs, 3271887872 bytes
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ error_code=124
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ set -e
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' 124 -eq 124 ']'
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_pool_images cluster2 mirror 50
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image_count=50
2024-05-17T15:16:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:16:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local count
2024-05-17T15:16:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_count=0
2024-05-17T15:16:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:16:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 40
2024-05-17T15:16:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in `seq 1 40`
2024-05-17T15:16:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 1 -ne 1
2024-05-17T15:16:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status
2024-05-17T15:16:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep 'images: '
2024-05-17T15:16:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cut '-d ' -f 2
2024-05-17T15:16:44.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: failed to query services: (13) Permission denied
2024-05-17T15:16:44.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ count=50
2024-05-17T15:16:44.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 50 = 50
2024-05-17T15:16:44.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_pool_healthy cluster2 mirror
2024-05-17T15:16:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:16:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:16:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state
2024-05-17T15:16:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 40
2024-05-17T15:16:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in `seq 1 40`
2024-05-17T15:16:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 1 -ne 1
2024-05-17T15:16:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status
2024-05-17T15:16:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep 'image health:'
2024-05-17T15:16:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cut '-d ' -f 3
2024-05-17T15:16:44.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: failed to query services: (13) Permission denied
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=OK
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test OK = ERROR
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test OK = OK
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_pool_images cluster1 mirror 50
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image_count=50
2024-05-17T15:16:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:16:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local count
2024-05-17T15:16:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local last_count=0
2024-05-17T15:16:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:16:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 40
2024-05-17T15:16:44.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in `seq 1 40`
2024-05-17T15:16:44.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 1 -ne 1
2024-05-17T15:16:44.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status
2024-05-17T15:16:44.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep 'images: '
2024-05-17T15:16:44.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cut '-d ' -f 2
2024-05-17T15:16:44.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: failed to query services: (13) Permission denied
2024-05-17T15:16:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ count=50
2024-05-17T15:16:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 50 = 50
2024-05-17T15:16:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:16:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_pool_healthy cluster1 mirror
2024-05-17T15:16:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:16:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:16:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:16:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state
2024-05-17T15:16:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 40
2024-05-17T15:16:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in `seq 1 40`
2024-05-17T15:16:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 1 -ne 1
2024-05-17T15:16:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status
2024-05-17T15:16:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep 'image health:'
2024-05-17T15:16:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cut '-d ' -f 3
2024-05-17T15:16:44.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: failed to query services: (13) Permission denied
2024-05-17T15:16:44.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=WARNING
2024-05-17T15:16:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test WARNING = ERROR
2024-05-17T15:16:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test WARNING = OK
2024-05-17T15:16:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in `seq 1 40`
2024-05-17T15:16:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 2 -ne 1
2024-05-17T15:16:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 30
2024-05-17T15:16:45.956 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~22s
2024-05-17T15:16:51.361 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~27s
2024-05-17T15:16:56.765 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~32s
2024-05-17T15:17:02.170 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~38s
2024-05-17T15:17:07.573 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~43s
2024-05-17T15:17:12.695 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:17:12.695+0000 7f075cc03640 -1 reset not still connected to 0x55ec6e959ee0
2024-05-17T15:17:12.696 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:17:12.695+0000 7f075cc03640 -1 reset not still connected to 0x55ec6ef971e0
2024-05-17T15:17:12.979 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~49s
2024-05-17T15:17:14.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9789242547054466) > weight (0.25)
2024-05-17T15:17:14.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4836873398711946) > weight (0.25)
2024-05-17T15:17:14.054 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T15:17:14.055 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T15:17:14.055 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons
2024-05-17T15:17:14.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T15:17:14.056+0000 7f99de0a8640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 42105) UID: 1000
2024-05-17T15:17:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status
2024-05-17T15:17:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep 'image health:'
2024-05-17T15:17:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cut '-d ' -f 3
2024-05-17T15:17:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: failed to query services: (13) Permission denied
2024-05-17T15:17:14.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ state=OK
2024-05-17T15:17:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test OK = ERROR
2024-05-17T15:17:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test OK = OK
2024-05-17T15:17:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ testlog 'TEST: compare many images'
2024-05-17T15:17:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ tee -a /tmp/tmp.mfaeqiv6jj/rbd-mirror.test.log
2024-05-17T15:17:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ date '+%F %T'
2024-05-17T15:17:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo 2024-05-17 15:17:14 TEST: compare many images
2024-05-17T15:17:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-17 15:17:14 TEST: compare many images
2024-05-17T15:17:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ seq 1 50
2024-05-17T15:17:14.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_1
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_1 snap
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:17:16.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_1
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:17.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_1 started
2024-05-17T15:17:17.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:17.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:17.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:17.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:17:17.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:17:17.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:17:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:17:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:17:17.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:17:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:17:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:17:17.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:17.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:17.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:17.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1
2024-05-17T15:17:17.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_1']
2024-05-17T15:17:17.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:17:17.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:17:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:17:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:17.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:17:17.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:17:17.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:17.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:17.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:17:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:17:17.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:17.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:17.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:17.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1
2024-05-17T15:17:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:17:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:17:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:17:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:17:18.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:17:18.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:17:18.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:17:18.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_1
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_1
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:18.054 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_1.status
2024-05-17T15:17:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_1.status
2024-05-17T15:17:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.status
2024-05-17T15:17:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_1.status
2024-05-17T15:17:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_1.status
2024-05-17T15:17:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:18.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:18.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_1
2024-05-17T15:17:18.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:18.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:18.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:18.059 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1
2024-05-17T15:17:18.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~54s
2024-05-17T15:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 86
2024-05-17T15:17:18.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.status
2024-05-17T15:17:18.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:18.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:18.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:18.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.status
2024-05-17T15:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml
2024-05-17T15:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=86
2024-05-17T15:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:17:19.032 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:19.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:19.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:19.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:19.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:19.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml
2024-05-17T15:17:19.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 86 = ''
2024-05-17T15:17:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:17:19.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:19.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:19.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:19.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:19.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:19.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml
2024-05-17T15:17:19.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:19.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:19.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:19.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:19.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:19.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 86 = ''
2024-05-17T15:17:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:17:20.056 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T15:17:20.056 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:17:20.056 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T15:17:20.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T15:17:20.057 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T15:17:20.060 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T15:17:20.060 INFO:tasks.rbd_mirror_thrash:waiting for 97 secs before thrashing
2024-05-17T15:17:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:20.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.status
2024-05-17T15:17:20.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml
2024-05-17T15:17:20.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:20.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=86
2024-05-17T15:17:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 86 = 86
2024-05-17T15:17:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap
2024-05-17T15:17:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:17:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_1':
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220a2a23448
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220a2a23448
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:17:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:07:40 2024
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:07:40 2024
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:07:40 2024
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 99506823-3ae7-4ad4-91fc-3c5dfa1d75a2
2024-05-17T15:17:21.507 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_1 snap
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_1
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.export
2024-05-17T15:17:21.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.export
2024-05-17T15:17:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.export
2024-05-17T15:17:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.export
2024-05-17T15:17:21.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:21.722 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.export
2024-05-17T15:17:22.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:22.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.export
2024-05-17T15:17:22.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_1.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_1.export
2024-05-17T15:17:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:17:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_2
2024-05-17T15:17:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_2 snap
2024-05-17T15:17:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_2
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:17:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:23.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:23.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:23.787 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~59s
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_2 started
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:17:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2
2024-05-17T15:17:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:17:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:17:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:17:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:17:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:17:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:17:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:17:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:17:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2
2024-05-17T15:17:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_2']
2024-05-17T15:17:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:17:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:17:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:17:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:17:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:17:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2
2024-05-17T15:17:25.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:17:25.097 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:17:25.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:17:25.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:17:25.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:17:25.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:17:25.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_2
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_2
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:25.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_2.status
2024-05-17T15:17:25.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:25.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_2.status
2024-05-17T15:17:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.status
2024-05-17T15:17:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_2.status
2024-05-17T15:17:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_2.status
2024-05-17T15:17:25.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:25.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:25.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_2
2024-05-17T15:17:25.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:25.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:25.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:25.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2
2024-05-17T15:17:25.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 88
2024-05-17T15:17:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.status
2024-05-17T15:17:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.status
2024-05-17T15:17:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml
2024-05-17T15:17:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:17:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=88
2024-05-17T15:17:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:17:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:26.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml
2024-05-17T15:17:26.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:26.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:26.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:26.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:26.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:26.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:26.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:26.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 88 = ''
2024-05-17T15:17:26.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:26.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:17:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml
2024-05-17T15:17:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:26.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:26.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 88 = ''
2024-05-17T15:17:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:17:27.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:27.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:27.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:27.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.status
2024-05-17T15:17:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml
2024-05-17T15:17:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:27.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=88
2024-05-17T15:17:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 88 = 88
2024-05-17T15:17:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:17:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:28.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap
2024-05-17T15:17:28.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_2':
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220db8d10dd
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220db8d10dd
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:07:56 2024
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:07:56 2024
2024-05-17T15:17:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:07:56 2024
2024-05-17T15:17:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:17:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:17:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:17:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 4ca271f9-b125-44e2-b0c0-f98df8231840
2024-05-17T15:17:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_2 snap
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_2
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.export
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.export
2024-05-17T15:17:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.export
2024-05-17T15:17:28.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.export
2024-05-17T15:17:28.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:28.796 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.export
2024-05-17T15:17:29.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:29.135 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.export
2024-05-17T15:17:29.190 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~65s
2024-05-17T15:17:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_2.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_2.export
2024-05-17T15:17:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:17:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_3
2024-05-17T15:17:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_3 snap
2024-05-17T15:17:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:17:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_3
2024-05-17T15:17:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_3 started
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:17:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:17:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3
2024-05-17T15:17:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:17:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:17:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:17:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:17:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:17:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:17:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:17:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:17:31.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:17:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:31.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:31.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:31.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:31.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:17:31.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:17:31.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:17:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3
2024-05-17T15:17:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_3']
2024-05-17T15:17:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:17:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:17:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3
2024-05-17T15:17:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:17:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:17:31.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:31.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:17:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:17:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:17:31.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:31.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:31.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:31.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3
2024-05-17T15:17:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:17:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:17:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:17:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:17:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:17:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:17:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:17:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_3
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_3
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_3.status
2024-05-17T15:17:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_3.status
2024-05-17T15:17:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:32.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.status
2024-05-17T15:17:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_3.status
2024-05-17T15:17:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_3.status
2024-05-17T15:17:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:32.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_3
2024-05-17T15:17:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3
2024-05-17T15:17:32.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 90
2024-05-17T15:17:32.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.status
2024-05-17T15:17:32.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:32.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:32.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:32.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.status
2024-05-17T15:17:32.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml
2024-05-17T15:17:32.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:32.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:17:32.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=90
2024-05-17T15:17:32.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:32.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:32.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:17:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml
2024-05-17T15:17:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:33.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:33.164 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 90 = ''
2024-05-17T15:17:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:17:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml
2024-05-17T15:17:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:33.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 90 = ''
2024-05-17T15:17:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:17:34.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:34.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:34.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:34.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:34.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.status
2024-05-17T15:17:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml
2024-05-17T15:17:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:34.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=90
2024-05-17T15:17:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 90 = 90
2024-05-17T15:17:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:17:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:34.594 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~70s
2024-05-17T15:17:35.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_3':
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220983d1372
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220983d1372
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:11 2024
2024-05-17T15:17:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:11 2024
2024-05-17T15:17:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:11 2024
2024-05-17T15:17:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:17:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:17:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:17:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 5814ab80-a23a-4a60-ba3c-ddb2e8964a09
2024-05-17T15:17:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:17:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_3 snap
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_3
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.export
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.export
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.export
2024-05-17T15:17:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.export
2024-05-17T15:17:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.export
2024-05-17T15:17:36.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.export
2024-05-17T15:17:36.221 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_3.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_3.export
2024-05-17T15:17:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:17:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_4
2024-05-17T15:17:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_4 snap
2024-05-17T15:17:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:17:37.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_4
2024-05-17T15:17:37.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:38.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_4 started
2024-05-17T15:17:38.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:38.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:38.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:38.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:17:38.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:17:38.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:17:38.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:17:38.491 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:17:38.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:17:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:17:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:17:38.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:38.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:38.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:38.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4
2024-05-17T15:17:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_4']
2024-05-17T15:17:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:17:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:17:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:38.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:17:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:17:38.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:38.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:38.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:38.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4
2024-05-17T15:17:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:17:38.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:17:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:17:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:17:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:17:38.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:17:38.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:17:38.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:38.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_4
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_4
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:38.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_4.status
2024-05-17T15:17:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_4.status
2024-05-17T15:17:38.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:38.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.status
2024-05-17T15:17:38.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_4.status
2024-05-17T15:17:38.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_4.status
2024-05-17T15:17:38.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:38.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:38.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_4
2024-05-17T15:17:38.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:38.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:38.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:38.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4
2024-05-17T15:17:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 92
2024-05-17T15:17:39.486 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.484+0000 7f28b79e2640 -1 reset not still connected to 0x5625bede5ad0
2024-05-17T15:17:39.486 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.484+0000 7f28b79e2640 -1 reset not still connected to 0x5625bf12e000
2024-05-17T15:17:39.494 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625c3ee09c0
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625c75f3930
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625cb513450
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625cb7e1860
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625cd1b5790
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625cf1e0dd0
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625cf878f70
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625d4277ad0
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625d966fc70
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625db04e270
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625e3086ea0
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625e3bdd790
2024-05-17T15:17:39.495 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625e6542c30
2024-05-17T15:17:39.496 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625eedbcc30
2024-05-17T15:17:39.496 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625eedbcf70
2024-05-17T15:17:39.496 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625eee089c0
2024-05-17T15:17:39.496 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:17:39.492+0000 7f28b79e2640 -1 reset not still connected to 0x5625f5157c70
2024-05-17T15:17:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.status
2024-05-17T15:17:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.status
2024-05-17T15:17:39.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml
2024-05-17T15:17:39.506 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:17:39.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=92
2024-05-17T15:17:39.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:39.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:39.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:17:39.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:39.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:39.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:39.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:39.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:39.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml
2024-05-17T15:17:39.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:39.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:39.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=38
2024-05-17T15:17:39.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 92 = 38
2024-05-17T15:17:39.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:39.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:17:39.997 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~76s
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml
2024-05-17T15:17:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:40.304 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:40.305 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:40.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:40.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:40.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 92 = ''
2024-05-17T15:17:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:17:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml
2024-05-17T15:17:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:41.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:41.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 92 = ''
2024-05-17T15:17:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:41.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:17:42.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.status
2024-05-17T15:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml
2024-05-17T15:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:42.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:42.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=92
2024-05-17T15:17:42.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 92 = 92
2024-05-17T15:17:42.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:42.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:17:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:43.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap
2024-05-17T15:17:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_4':
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12203b21a97a
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12203b21a97a
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:26 2024
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:26 2024
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:26 2024
2024-05-17T15:17:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:17:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:17:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:17:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 5c3f1454-afcf-43a9-8f75-0f06d2c9f10d
2024-05-17T15:17:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_4 snap
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_4
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.export
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.export
2024-05-17T15:17:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.export
2024-05-17T15:17:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.export
2024-05-17T15:17:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.export
2024-05-17T15:17:44.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:44.450 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.export
2024-05-17T15:17:44.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_4.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_4.export
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_5
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_5 snap
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:17:45.402 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~81s
2024-05-17T15:17:46.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_5
2024-05-17T15:17:46.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:46.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started
2024-05-17T15:17:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:17:46.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:46.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:46.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_5 started
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:17:47.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:17:47.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5
2024-05-17T15:17:47.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:17:47.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:47.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:17:47.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:17:47.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:17:47.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:17:47.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:47.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:17:47.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:17:47.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:17:47.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:47.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:17:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:47.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:47.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:47.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:17:47.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:17:47.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:17:47.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5
2024-05-17T15:17:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_5']
2024-05-17T15:17:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:17:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:17:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5
2024-05-17T15:17:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:17:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:17:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:17:47.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:47.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:47.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:47.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5
2024-05-17T15:17:47.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:17:47.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:17:47.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:17:47.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_5
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_5
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_5.status
2024-05-17T15:17:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_5.status
2024-05-17T15:17:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.status
2024-05-17T15:17:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_5.status
2024-05-17T15:17:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_5.status
2024-05-17T15:17:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_5
2024-05-17T15:17:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5
2024-05-17T15:17:48.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 94
2024-05-17T15:17:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.status
2024-05-17T15:17:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.status
2024-05-17T15:17:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml
2024-05-17T15:17:48.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:48.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:17:48.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=94
2024-05-17T15:17:48.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:48.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:48.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml
2024-05-17T15:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=39
2024-05-17T15:17:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 94 = 39
2024-05-17T15:17:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml
2024-05-17T15:17:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:49.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:49.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 94 = ''
2024-05-17T15:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:17:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml
2024-05-17T15:17:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 94 = ''
2024-05-17T15:17:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:17:50.806 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~86s
2024-05-17T15:17:51.074 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:17:51.075+0000 7fc15d65d640 -1 reset not still connected to 0x55fd123e09c0
2024-05-17T15:17:51.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:51.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:51.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:51.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:51.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.status
2024-05-17T15:17:51.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml
2024-05-17T15:17:51.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:51.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=94
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 94 = 94
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap
2024-05-17T15:17:53.092 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_5':
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220458afe22
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220458afe22
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:17:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:34 2024
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:34 2024
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:34 2024
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 6a7fbf74-3f03-4071-9e4a-10ba83516c52
2024-05-17T15:17:53.094 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:17:53.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:53.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_5 snap
2024-05-17T15:17:53.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:53.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_5
2024-05-17T15:17:53.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:53.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:17:53.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.export
2024-05-17T15:17:53.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.export
2024-05-17T15:17:53.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.export
2024-05-17T15:17:53.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.export
2024-05-17T15:17:53.319 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:53.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.export
2024-05-17T15:17:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:17:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.export
2024-05-17T15:17:53.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_5.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_5.export
2024-05-17T15:17:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:17:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_6
2024-05-17T15:17:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_6 snap
2024-05-17T15:17:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_6
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:55.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:55.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:17:56.210 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~92s
2024-05-17T15:17:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_6 started
2024-05-17T15:17:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:17:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:17:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:17:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:17:56.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:17:56.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:17:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:17:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:17:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:17:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:56.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:56.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:17:56.664 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6
2024-05-17T15:17:56.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_6']
2024-05-17T15:17:56.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:17:56.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:17:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:17:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:17:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:17:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:17:56.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:56.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:56.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:17:56.802 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6
2024-05-17T15:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:17:56.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:17:56.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:17:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_6
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_6
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_6.status
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_6.status
2024-05-17T15:17:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:56.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.status
2024-05-17T15:17:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_6.status
2024-05-17T15:17:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_6.status
2024-05-17T15:17:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:17:56.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:56.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_6
2024-05-17T15:17:56.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:56.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:56.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:56.974 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6
2024-05-17T15:17:57.382 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T15:17:57.382+0000 7fd07b53d640 -1 reset not still connected to 0x5577f21036c0
2024-05-17T15:17:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 96
2024-05-17T15:17:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.status
2024-05-17T15:17:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:17:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.status
2024-05-17T15:17:57.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml
2024-05-17T15:17:57.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:57.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:17:57.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=96
2024-05-17T15:17:57.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:57.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:57.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:17:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:57.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml
2024-05-17T15:17:57.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:58.000 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:58.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 96 = ''
2024-05-17T15:17:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:17:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml
2024-05-17T15:17:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:58.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:17:58.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:17:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:17:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:17:58.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:17:58.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 96 = ''
2024-05-17T15:17:58.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:17:58.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:17:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.status
2024-05-17T15:17:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml
2024-05-17T15:17:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:17:59.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=96
2024-05-17T15:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 96 = 96
2024-05-17T15:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap
2024-05-17T15:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap
2024-05-17T15:18:00.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_6':
2024-05-17T15:18:00.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12202a7600c5
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12202a7600c5
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:48 2024
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:48 2024
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:48 2024
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:00.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:00.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:00.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7aeb365d-f128-4cac-a752-c8e9bc4d1961
2024-05-17T15:18:00.405 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_6 snap
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_6
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.export
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.export
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.export
2024-05-17T15:18:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.export
2024-05-17T15:18:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:00.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.export
2024-05-17T15:18:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.export
2024-05-17T15:18:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_6.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_6.export
2024-05-17T15:18:01.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:01.028 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_7
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_7 snap
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:01.614 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~97s
2024-05-17T15:18:02.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_7
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_7 started
2024-05-17T15:18:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:03.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7
2024-05-17T15:18:03.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_7']
2024-05-17T15:18:03.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:03.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:03.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:03.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:03.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:03.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:03.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:03.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:03.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:03.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7
2024-05-17T15:18:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:03.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:03.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:03.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:03.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_7
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_7
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:03.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:03.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_7.status
2024-05-17T15:18:03.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_7.status
2024-05-17T15:18:03.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:03.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.status
2024-05-17T15:18:03.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_7.status
2024-05-17T15:18:03.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_7.status
2024-05-17T15:18:03.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_7
2024-05-17T15:18:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7
2024-05-17T15:18:04.052 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6529188702381534) > weight (0.25)
2024-05-17T15:18:04.052 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.787229584842409) > weight (0.25)
2024-05-17T15:18:04.052 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5621664952998223) > weight (0.25)
2024-05-17T15:18:04.052 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7689966733553241) > weight (0.25)
2024-05-17T15:18:04.052 INFO:tasks.rbd_mirror_thrash:waiting for 86 secs before thrashing
2024-05-17T15:18:04.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 98
2024-05-17T15:18:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.status
2024-05-17T15:18:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:04.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:04.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:04.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.status
2024-05-17T15:18:04.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml
2024-05-17T15:18:04.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:04.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=98
2024-05-17T15:18:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:04.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml
2024-05-17T15:18:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:05.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:05.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:05.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:05.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:05.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:05.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:05.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 98 = ''
2024-05-17T15:18:05.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:05.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:05.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml
2024-05-17T15:18:05.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:05.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:05.492 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 98 = ''
2024-05-17T15:18:05.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:05.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:06.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:06.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml
2024-05-17T15:18:06.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:06.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:06.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:06.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:06.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:06.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:06.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:06.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 98 = ''
2024-05-17T15:18:06.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:06.355 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:18:07.018 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~103s
2024-05-17T15:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.status
2024-05-17T15:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml
2024-05-17T15:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=98
2024-05-17T15:18:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 98 = 98
2024-05-17T15:18:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap
2024-05-17T15:18:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:09.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap
2024-05-17T15:18:09.098 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_7':
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12207e43a81c
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12207e43a81c
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:08:59 2024
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:08:59 2024
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:08:59 2024
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 26533277-dd10-458b-9fe7-f5ca55df003d
2024-05-17T15:18:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_7 snap
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_7
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.export
2024-05-17T15:18:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.export
2024-05-17T15:18:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.export
2024-05-17T15:18:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.export
2024-05-17T15:18:09.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.export
2024-05-17T15:18:09.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:09.628 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.export
2024-05-17T15:18:09.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_7.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_7.export
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_8
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_8 snap
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:09.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:09.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:11.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_8
2024-05-17T15:18:11.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:11.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:11.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:11.735 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:12.422 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~108s
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_8 started
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:12.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:12.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8
2024-05-17T15:18:12.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:12.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:12.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:12.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:12.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:12.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:12.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:12.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:12.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:12.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8
2024-05-17T15:18:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_8']
2024-05-17T15:18:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:12.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:12.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8
2024-05-17T15:18:12.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:12.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:12.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:12.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:12.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8
2024-05-17T15:18:13.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:13.044 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_8
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_8
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_8.status
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_8.status
2024-05-17T15:18:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.status
2024-05-17T15:18:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_8.status
2024-05-17T15:18:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_8.status
2024-05-17T15:18:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_8
2024-05-17T15:18:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8
2024-05-17T15:18:13.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 100
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.status
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.status
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml
2024-05-17T15:18:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=100
2024-05-17T15:18:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:14.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:14.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:14.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:14.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:14.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml
2024-05-17T15:18:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:14.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:14.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 100 = ''
2024-05-17T15:18:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:14.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml
2024-05-17T15:18:14.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:14.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:14.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 100 = ''
2024-05-17T15:18:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:14.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:15.390 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:15.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:15.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:15.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:15.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.status
2024-05-17T15:18:15.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml
2024-05-17T15:18:15.391 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=100
2024-05-17T15:18:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 100 = 100
2024-05-17T15:18:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:16.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_8':
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 122045c8273d
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.122045c8273d
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:13 2024
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:13 2024
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:13 2024
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: f76cd062-3e00-458d-a951-27acd55808b3
2024-05-17T15:18:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_8 snap
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_8
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.export
2024-05-17T15:18:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.export
2024-05-17T15:18:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.export
2024-05-17T15:18:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.export
2024-05-17T15:18:16.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:16.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.export
2024-05-17T15:18:17.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.export
2024-05-17T15:18:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_8.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_8.export
2024-05-17T15:18:17.159 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:17.159 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_9
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_9 snap
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:17.826 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~113s
2024-05-17T15:18:18.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_9
2024-05-17T15:18:18.769 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_9 started
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9
2024-05-17T15:18:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:19.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:19.774 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:19.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:19.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:19.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:19.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9
2024-05-17T15:18:19.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_9']
2024-05-17T15:18:19.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:19.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:19.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:19.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9
2024-05-17T15:18:20.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:20.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:20.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:20.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:20.086 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:20.087 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_9
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_9
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:20.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_9.status
2024-05-17T15:18:20.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_9.status
2024-05-17T15:18:20.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.status
2024-05-17T15:18:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_9.status
2024-05-17T15:18:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_9.status
2024-05-17T15:18:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_9
2024-05-17T15:18:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9
2024-05-17T15:18:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 102
2024-05-17T15:18:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.status
2024-05-17T15:18:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.status
2024-05-17T15:18:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml
2024-05-17T15:18:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:20.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=102
2024-05-17T15:18:20.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:20.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:20.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml
2024-05-17T15:18:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:21.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:21.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 102 = ''
2024-05-17T15:18:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml
2024-05-17T15:18:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:21.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:21.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:21.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:21.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:21.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:21.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:21.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 102 = ''
2024-05-17T15:18:21.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:21.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:22.221 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.219+0000 7f28b79e2640 -1 reset not still connected to 0x5625bede5ad0
2024-05-17T15:18:22.236 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.235+0000 7f28b79e2640 -1 reset not still connected to 0x5625bf12e000
2024-05-17T15:18:22.238 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.235+0000 7f28b79e2640 -1 reset not still connected to 0x5625c4ce0000
2024-05-17T15:18:22.238 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.235+0000 7f28b79e2640 -1 reset not still connected to 0x5625c75f3930
2024-05-17T15:18:22.238 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.235+0000 7f28b79e2640 -1 reset not still connected to 0x5625cb513450
2024-05-17T15:18:22.251 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.247+0000 7f28b79e2640 -1 reset not still connected to 0x5625cd1b5790
2024-05-17T15:18:22.251 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.247+0000 7f28b79e2640 -1 reset not still connected to 0x5625cf1e0dd0
2024-05-17T15:18:22.251 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.247+0000 7f28b79e2640 -1 reset not still connected to 0x5625cf878f70
2024-05-17T15:18:22.257 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.255+0000 7f28b79e2640 -1 reset not still connected to 0x5625d4277ad0
2024-05-17T15:18:22.258 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.255+0000 7f28b79e2640 -1 reset not still connected to 0x5625d966fc70
2024-05-17T15:18:22.258 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.255+0000 7f28b79e2640 -1 reset not still connected to 0x5625db04e270
2024-05-17T15:18:22.258 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.255+0000 7f28b79e2640 -1 reset not still connected to 0x5625e3086ea0
2024-05-17T15:18:22.258 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.255+0000 7f28b79e2640 -1 reset not still connected to 0x5625e3bdd790
2024-05-17T15:18:22.262 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625e6542c30
2024-05-17T15:18:22.262 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625eedbcc30
2024-05-17T15:18:22.262 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625eedbcf70
2024-05-17T15:18:22.262 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625eee089c0
2024-05-17T15:18:22.262 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625f0d2dad0
2024-05-17T15:18:22.263 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625f5157c70
2024-05-17T15:18:22.263 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:18:22.259+0000 7f28b79e2640 -1 reset not still connected to 0x5625fdb51930
2024-05-17T15:18:22.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.status
2024-05-17T15:18:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml
2024-05-17T15:18:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=102
2024-05-17T15:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 102 = 102
2024-05-17T15:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap
2024-05-17T15:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:22.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:22.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:22.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:22.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:22.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:23.230 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~119s
2024-05-17T15:18:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_9':
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220c0c21c2b
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220c0c21c2b
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:23.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:27 2024
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:27 2024
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:27 2024
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 128beb1d-d528-4574-85e6-dc2e3922fc06
2024-05-17T15:18:23.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_9 snap
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_9
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.export
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.export
2024-05-17T15:18:23.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.export
2024-05-17T15:18:23.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.export
2024-05-17T15:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:23.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.export
2024-05-17T15:18:24.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.export
2024-05-17T15:18:24.231 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_9.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_9.export
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_10
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_10 snap
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:24.273 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:25.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_10
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:25.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_10 started
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:26.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:26.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:26.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:26.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10
2024-05-17T15:18:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_10']
2024-05-17T15:18:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:26.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:26.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:26.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:26.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:26.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:26.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10
2024-05-17T15:18:27.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:27.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:27.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:27.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:27.144 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_10
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_10
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_10.status
2024-05-17T15:18:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_10.status
2024-05-17T15:18:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.status
2024-05-17T15:18:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_10.status
2024-05-17T15:18:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_10.status
2024-05-17T15:18:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_10
2024-05-17T15:18:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10
2024-05-17T15:18:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 104
2024-05-17T15:18:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.status
2024-05-17T15:18:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.status
2024-05-17T15:18:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml
2024-05-17T15:18:27.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=104
2024-05-17T15:18:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:28.101 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml
2024-05-17T15:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:28.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:28.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:28.174 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:28.174 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 104 = ''
2024-05-17T15:18:28.174 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:28.174 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:28.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml
2024-05-17T15:18:28.575 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:28.633 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~124s
2024-05-17T15:18:28.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:28.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 104 = ''
2024-05-17T15:18:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:29.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:29.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.status
2024-05-17T15:18:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml
2024-05-17T15:18:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:29.518 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:29.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=104
2024-05-17T15:18:29.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 104 = 104
2024-05-17T15:18:29.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap
2024-05-17T15:18:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_10':
2024-05-17T15:18:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220308f1547
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220308f1547
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:42 2024
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:42 2024
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:42 2024
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 6a55ffff-b717-4cdd-9603-20c25dd3ae1c
2024-05-17T15:18:30.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_10 snap
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_10
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.export
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.export
2024-05-17T15:18:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.export
2024-05-17T15:18:30.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.export
2024-05-17T15:18:30.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:30.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.export
2024-05-17T15:18:31.207 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.export
2024-05-17T15:18:31.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_10.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_10.export
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_11
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_11 snap
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:31.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:32.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_11
2024-05-17T15:18:32.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:32.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:33.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_11 started
2024-05-17T15:18:33.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:33.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:33.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:33.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:33.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:33.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:33.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:33.514 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11
2024-05-17T15:18:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:33.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:33.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:33.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:33.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11
2024-05-17T15:18:33.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_11']
2024-05-17T15:18:33.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:33.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11
2024-05-17T15:18:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11
2024-05-17T15:18:33.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:33.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:33.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:33.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:33.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:33.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:33.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:33.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:33.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_11
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_11
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:33.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_11.status
2024-05-17T15:18:33.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_11.status
2024-05-17T15:18:33.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:33.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.status
2024-05-17T15:18:33.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_11.status
2024-05-17T15:18:33.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_11.status
2024-05-17T15:18:33.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:33.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_11
2024-05-17T15:18:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11
2024-05-17T15:18:34.037 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~130s
2024-05-17T15:18:34.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 106
2024-05-17T15:18:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.status
2024-05-17T15:18:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.status
2024-05-17T15:18:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml
2024-05-17T15:18:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=106
2024-05-17T15:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:34.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:34.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:34.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:34.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:34.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:34.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml
2024-05-17T15:18:34.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:34.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=45
2024-05-17T15:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 106 = 45
2024-05-17T15:18:34.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:34.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:35.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:35.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:35.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:35.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:35.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:35.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml
2024-05-17T15:18:35.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:35.350 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:35.350 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 106 = ''
2024-05-17T15:18:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml
2024-05-17T15:18:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 106 = ''
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:18:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.status
2024-05-17T15:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml
2024-05-17T15:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:38.013 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=106
2024-05-17T15:18:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 106 = 106
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:39.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap
2024-05-17T15:18:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_11':
2024-05-17T15:18:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220e4d55740
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220e4d55740
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:09:52 2024
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:09:52 2024
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:09:52 2024
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:39.080 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ccec9d36-68f0-4c69-b480-a47cd5e5df24
2024-05-17T15:18:39.081 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_11 snap
2024-05-17T15:18:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_11
2024-05-17T15:18:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.export
2024-05-17T15:18:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.export
2024-05-17T15:18:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.export
2024-05-17T15:18:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.export
2024-05-17T15:18:39.291 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:39.300 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.export
2024-05-17T15:18:39.441 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~135s
2024-05-17T15:18:39.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.export
2024-05-17T15:18:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_11.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_11.export
2024-05-17T15:18:39.786 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_12
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_12 snap
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:39.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_12
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:42.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_12 started
2024-05-17T15:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:42.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:42.548 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:42.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:42.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12
2024-05-17T15:18:42.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_12']
2024-05-17T15:18:42.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:42.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:42.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:42.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:42.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:42.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:42.714 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12
2024-05-17T15:18:42.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:42.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_12
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_12
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_12.status
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_12.status
2024-05-17T15:18:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.status
2024-05-17T15:18:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_12.status
2024-05-17T15:18:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_12.status
2024-05-17T15:18:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:42.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:42.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_12
2024-05-17T15:18:42.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:42.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:42.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:42.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12
2024-05-17T15:18:43.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 108
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.status
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.status
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml
2024-05-17T15:18:43.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:43.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=108
2024-05-17T15:18:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:43.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:43.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:43.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:43.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:43.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml
2024-05-17T15:18:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:43.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:43.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=46
2024-05-17T15:18:43.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 108 = 46
2024-05-17T15:18:43.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:43.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:44.314 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml
2024-05-17T15:18:44.314 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 108 = ''
2024-05-17T15:18:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:44.398 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:44.845 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~141s
2024-05-17T15:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:45.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:45.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.status
2024-05-17T15:18:45.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml
2024-05-17T15:18:45.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=108
2024-05-17T15:18:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 108 = 108
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:45.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:46.351 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_12':
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220ff5a4c38
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220ff5a4c38
2024-05-17T15:18:46.419 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:00 2024
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:00 2024
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:00 2024
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 915c9e6a-ae95-4135-a900-860abb54b53e
2024-05-17T15:18:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_12 snap
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_12
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.export
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.export
2024-05-17T15:18:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.export
2024-05-17T15:18:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.export
2024-05-17T15:18:46.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:46.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.export
2024-05-17T15:18:46.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:46.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.export
2024-05-17T15:18:47.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_12.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_12.export
2024-05-17T15:18:47.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:47.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:47.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_13
2024-05-17T15:18:47.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_13 snap
2024-05-17T15:18:47.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:47.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:47.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:47.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:47.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:48.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_13
2024-05-17T15:18:48.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:48.558 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_13 started
2024-05-17T15:18:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:49.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:49.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:49.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:49.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13
2024-05-17T15:18:49.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:49.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:49.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:49.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:49.568 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:49.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:49.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:49.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13
2024-05-17T15:18:49.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_13']
2024-05-17T15:18:49.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:49.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:49.715 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13
2024-05-17T15:18:49.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:49.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:49.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:49.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:49.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_13
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_13
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_13.status
2024-05-17T15:18:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_13.status
2024-05-17T15:18:49.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:49.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.status
2024-05-17T15:18:49.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_13.status
2024-05-17T15:18:49.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_13.status
2024-05-17T15:18:49.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:49.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:49.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_13
2024-05-17T15:18:49.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:49.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:49.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:49.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13
2024-05-17T15:18:50.250 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~146s
2024-05-17T15:18:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 110
2024-05-17T15:18:50.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.status
2024-05-17T15:18:50.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:50.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:50.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:50.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.status
2024-05-17T15:18:50.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml
2024-05-17T15:18:50.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:50.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:50.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=110
2024-05-17T15:18:50.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:50.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:50.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:18:50.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:50.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:50.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:50.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:50.862 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:50.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml
2024-05-17T15:18:50.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=47
2024-05-17T15:18:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 110 = 47
2024-05-17T15:18:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:18:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml
2024-05-17T15:18:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:51.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 110 = ''
2024-05-17T15:18:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml
2024-05-17T15:18:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:18:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:18:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:18:52.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:18:52.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 110 = ''
2024-05-17T15:18:52.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:52.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:18:53.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:53.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:53.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:53.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:53.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.status
2024-05-17T15:18:53.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml
2024-05-17T15:18:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:53.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:18:53.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=110
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 110 = 110
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap
2024-05-17T15:18:55.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_13':
2024-05-17T15:18:55.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:18:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:18:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:18:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220392b2733
2024-05-17T15:18:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220392b2733
2024-05-17T15:18:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:18:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:18:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:18:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:18:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:13 2024
2024-05-17T15:18:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:13 2024
2024-05-17T15:18:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:13 2024
2024-05-17T15:18:55.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:18:55.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:18:55.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:18:55.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 2c4982c8-61ec-4697-b3aa-46a9ce03b746
2024-05-17T15:18:55.244 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:18:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_13 snap
2024-05-17T15:18:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_13
2024-05-17T15:18:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:18:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.export
2024-05-17T15:18:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.export
2024-05-17T15:18:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.export
2024-05-17T15:18:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.export
2024-05-17T15:18:55.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:55.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.export
2024-05-17T15:18:55.655 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~151s
2024-05-17T15:18:55.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:18:55.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.export
2024-05-17T15:18:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_13.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_13.export
2024-05-17T15:18:55.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:55.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_14
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_14 snap
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:18:55.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:18:57.061 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.39390228385372816) > weight (0.25)
2024-05-17T15:18:57.061 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.871869274054193) > weight (0.25)
2024-05-17T15:18:57.061 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4214968409142915) > weight (0.25)
2024-05-17T15:18:57.062 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3
2024-05-17T15:18:57.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15
2024-05-17T15:18:57.062 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons
2024-05-17T15:18:57.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi171.stderr:2024-05-17T15:18:57.062+0000 7fdecd895640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3  (PID: 35367) UID: 1000
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_14
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:18:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:18:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_14 started
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:18:58.670 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:18:58.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14
2024-05-17T15:18:58.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:18:58.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:18:58.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:18:58.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:18:58.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:58.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14
2024-05-17T15:18:58.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:18:58.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:58.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:18:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:18:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:18:58.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:58.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:58.701 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:18:58.702 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14
2024-05-17T15:18:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_14']
2024-05-17T15:18:58.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:18:58.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:18:58.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:58.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:18:58.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:18:58.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:18:58.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:18:58.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:18:58.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:18:58.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:18:58.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:18:58.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:18:58.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:58.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:58.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:18:58.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14
2024-05-17T15:18:58.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:18:58.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:18:58.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:18:58.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:18:58.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_14
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_14
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:58.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_14.status
2024-05-17T15:18:58.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_14.status
2024-05-17T15:18:58.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:59.000 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.status
2024-05-17T15:18:59.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_14.status
2024-05-17T15:18:59.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_14.status
2024-05-17T15:18:59.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:18:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:18:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_14
2024-05-17T15:18:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14
2024-05-17T15:18:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 112
2024-05-17T15:18:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.status
2024-05-17T15:18:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:18:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:18:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:18:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.status
2024-05-17T15:18:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml
2024-05-17T15:18:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:18:59.972 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:18:59.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=112
2024-05-17T15:18:59.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:18:59.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:18:59.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:00.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:19:00.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:00.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:00.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:19:00.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:19:00.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml
2024-05-17T15:19:00.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:00.250 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:00.250 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:00.252 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 112 = ''
2024-05-17T15:19:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:00.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:19:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:19:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:19:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml
2024-05-17T15:19:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:00.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:00.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:00.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:00.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:00.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:00.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:00.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 112 = ''
2024-05-17T15:19:00.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:00.729 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:01.058 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~157s
2024-05-17T15:19:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:19:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:19:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.status
2024-05-17T15:19:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml
2024-05-17T15:19:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=112
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 112 = 112
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:01.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:02.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_14':
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12209d1be457
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12209d1be457
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:28 2024
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:28 2024
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:28 2024
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 0b9e4c5c-db1d-41ce-a537-9dcae0dfcc10
2024-05-17T15:19:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:02.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_14 snap
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_14
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.export
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.export
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.export
2024-05-17T15:19:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.export
2024-05-17T15:19:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.export
2024-05-17T15:19:03.462 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.export
2024-05-17T15:19:03.556 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_14.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_14.export
2024-05-17T15:19:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_15
2024-05-17T15:19:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_15 snap
2024-05-17T15:19:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:04.063 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3
2024-05-17T15:19:04.063 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:19:04.064 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3
2024-05-17T15:19:04.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon
2024-05-17T15:19:04.064 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3
2024-05-17T15:19:04.067 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started
2024-05-17T15:19:04.067 INFO:tasks.rbd_mirror_thrash:waiting for 70 secs before thrashing
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_15
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_15 started
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:05.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15
2024-05-17T15:19:05.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:05.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:05.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:05.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:05.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:05.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:05.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:05.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:05.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15
2024-05-17T15:19:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_15']
2024-05-17T15:19:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:05.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:05.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:05.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:05.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:05.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15
2024-05-17T15:19:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_15
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_15
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:06.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_15.status
2024-05-17T15:19:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_15.status
2024-05-17T15:19:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.status
2024-05-17T15:19:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_15.status
2024-05-17T15:19:06.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_15.status
2024-05-17T15:19:06.077 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_15
2024-05-17T15:19:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15
2024-05-17T15:19:06.462 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~162s
2024-05-17T15:19:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 114
2024-05-17T15:19:06.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.status
2024-05-17T15:19:06.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.status
2024-05-17T15:19:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml
2024-05-17T15:19:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:06.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=114
2024-05-17T15:19:06.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:06.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:06.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:07.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:07.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:07.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:07.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:07.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:07.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml
2024-05-17T15:19:07.039 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:07.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:07.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:07.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:07.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:07.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:07.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 114 = ''
2024-05-17T15:19:07.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:07.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:07.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml
2024-05-17T15:19:07.513 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:07.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:07.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 114 = ''
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:07.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:08.385 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.status
2024-05-17T15:19:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml
2024-05-17T15:19:08.387 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:08.454 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:08.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=114
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 114 = 114
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:09.458 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap
2024-05-17T15:19:09.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_15':
2024-05-17T15:19:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220b7abbc76
2024-05-17T15:19:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220b7abbc76
2024-05-17T15:19:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:37 2024
2024-05-17T15:19:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:37 2024
2024-05-17T15:19:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:37 2024
2024-05-17T15:19:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 4963d81f-f6e8-40ba-b80c-6c7d9bd6f465
2024-05-17T15:19:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:09.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:09.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_15 snap
2024-05-17T15:19:09.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:09.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_15
2024-05-17T15:19:09.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:09.528 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:09.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.export
2024-05-17T15:19:09.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.export
2024-05-17T15:19:09.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.export
2024-05-17T15:19:09.529 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.export
2024-05-17T15:19:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:09.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.export
2024-05-17T15:19:10.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:10.090 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.export
2024-05-17T15:19:10.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_15.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_15.export
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_16
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_16 snap
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_16
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:11.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:11.782 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:11.865 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~168s
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_16 started
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:12.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16
2024-05-17T15:19:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_16']
2024-05-17T15:19:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:12.924 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:12.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:12.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16
2024-05-17T15:19:12.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:12.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:12.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:12.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:12.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16
2024-05-17T15:19:13.102 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:13.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:13.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:13.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:13.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_16
2024-05-17T15:19:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_16
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:13.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_16.status
2024-05-17T15:19:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_16.status
2024-05-17T15:19:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.status
2024-05-17T15:19:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_16.status
2024-05-17T15:19:13.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_16.status
2024-05-17T15:19:13.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_16
2024-05-17T15:19:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16
2024-05-17T15:19:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 116
2024-05-17T15:19:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.status
2024-05-17T15:19:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.status
2024-05-17T15:19:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml
2024-05-17T15:19:13.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:13.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:13.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=116
2024-05-17T15:19:13.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:13.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:13.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:14.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:14.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:14.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:14.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:14.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml
2024-05-17T15:19:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:14.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:14.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:14.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:14.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:14.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:14.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:14.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 116 = ''
2024-05-17T15:19:14.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:14.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:14.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:14.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:14.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:14.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:14.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml
2024-05-17T15:19:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:14.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:14.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 116 = ''
2024-05-17T15:19:14.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:14.622 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.status
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml
2024-05-17T15:19:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:15.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=116
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 116 = 116
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:15.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:15.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:16.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_16':
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12201f033d27
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12201f033d27
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:44 2024
2024-05-17T15:19:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:44 2024
2024-05-17T15:19:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:44 2024
2024-05-17T15:19:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: e802db82-1a1d-42f1-9c16-9e915554bd00
2024-05-17T15:19:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_16 snap
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_16
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.export
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.export
2024-05-17T15:19:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.export
2024-05-17T15:19:16.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.export
2024-05-17T15:19:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.export
2024-05-17T15:19:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:17.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.export
2024-05-17T15:19:17.268 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~173s
2024-05-17T15:19:17.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_16.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_16.export
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_17
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_17 snap
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:17.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_17
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:19.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_17 started
2024-05-17T15:19:19.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:19.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:19.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:19.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:19.804 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:19.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:19.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17
2024-05-17T15:19:19.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_17']
2024-05-17T15:19:19.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:19.939 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17
2024-05-17T15:19:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17
2024-05-17T15:19:20.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:20.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:20.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:20.114 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:20.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:20.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_17
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_17
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:20.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_17.status
2024-05-17T15:19:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_17.status
2024-05-17T15:19:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.status
2024-05-17T15:19:20.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_17.status
2024-05-17T15:19:20.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_17.status
2024-05-17T15:19:20.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_17
2024-05-17T15:19:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17
2024-05-17T15:19:20.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 118
2024-05-17T15:19:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.status
2024-05-17T15:19:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.status
2024-05-17T15:19:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml
2024-05-17T15:19:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:20.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:20.913 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=118
2024-05-17T15:19:20.913 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:20.913 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:20.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml
2024-05-17T15:19:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:21.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:21.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 118 = ''
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:21.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:21.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:21.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:21.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:21.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:21.594 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:21.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml
2024-05-17T15:19:21.595 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:21.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:21.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:21.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:21.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:21.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:21.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:21.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 118 = ''
2024-05-17T15:19:21.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:21.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:22.345 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.342+0000 7f905860c640 -1 reset not still connected to 0x55d26deb56c0
2024-05-17T15:19:22.353 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.350+0000 7f905860c640 -1 reset not still connected to 0x55d26fd0dad0
2024-05-17T15:19:22.353 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.350+0000 7f905860c640 -1 reset not still connected to 0x55d273413ad0
2024-05-17T15:19:22.354 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.350+0000 7f905860c640 -1 reset not still connected to 0x55d276c29ee0
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.354+0000 7f905860c640 -1 reset not still connected to 0x55d295556000
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.354+0000 7f905860c640 -1 reset not still connected to 0x55d29b35d1e0
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.354+0000 7f905860c640 -1 reset not still connected to 0x55d29c793520
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.354+0000 7f905860c640 -1 reset not still connected to 0x55d29e8fa9c0
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.354+0000 7f905860c640 -1 reset not still connected to 0x55d2a108dc70
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.354+0000 7f905860c640 -1 reset not still connected to 0x55d2a2848680
2024-05-17T15:19:22.364 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.358+0000 7f905860c640 -1 reset not still connected to 0x55d2a452ab60
2024-05-17T15:19:22.365 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:19:22.358+0000 7f905860c640 -1 reset not still connected to 0x55d2ac28c270
2024-05-17T15:19:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.status
2024-05-17T15:19:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml
2024-05-17T15:19:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:22.539 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:22.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=118
2024-05-17T15:19:22.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 118 = 118
2024-05-17T15:19:22.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:22.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:22.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:22.671 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~178s
2024-05-17T15:19:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_17':
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12207aa7eab5
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12207aa7eab5
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:10:55 2024
2024-05-17T15:19:23.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:10:55 2024
2024-05-17T15:19:23.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:10:55 2024
2024-05-17T15:19:23.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:23.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:23.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:23.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 51b71ae3-ecdd-405e-b221-6cbc4ee1b11e
2024-05-17T15:19:23.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:23.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:23.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_17 snap
2024-05-17T15:19:23.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_17
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.export
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.export
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.export
2024-05-17T15:19:23.611 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.export
2024-05-17T15:19:23.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:23.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.export
2024-05-17T15:19:24.215 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:24.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.export
2024-05-17T15:19:24.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_17.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_17.export
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_18
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_18 snap
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_18
2024-05-17T15:19:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_18 started
2024-05-17T15:19:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18
2024-05-17T15:19:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:26.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:26.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:26.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:26.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:26.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:26.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:26.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:26.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18
2024-05-17T15:19:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_18']
2024-05-17T15:19:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:27.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:27.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:27.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:27.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:27.022 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18
2024-05-17T15:19:27.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:27.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_18
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_18
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_18.status
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_18.status
2024-05-17T15:19:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:27.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.status
2024-05-17T15:19:27.200 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_18.status
2024-05-17T15:19:27.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_18.status
2024-05-17T15:19:27.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_18
2024-05-17T15:19:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18
2024-05-17T15:19:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 120
2024-05-17T15:19:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.status
2024-05-17T15:19:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.status
2024-05-17T15:19:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml
2024-05-17T15:19:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:27.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=120
2024-05-17T15:19:27.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:27.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:27.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:28.075 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~184s
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml
2024-05-17T15:19:28.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:28.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:28.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=52
2024-05-17T15:19:28.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 120 = 52
2024-05-17T15:19:28.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:28.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:28.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:28.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:28.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:28.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:28.630 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:28.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml
2024-05-17T15:19:28.631 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 120 = ''
2024-05-17T15:19:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml
2024-05-17T15:19:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:29.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:29.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 120 = ''
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:19:30.053 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0
2024-05-17T15:19:30.055 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15
2024-05-17T15:19:30.055 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons
2024-05-17T15:19:30.056 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T15:19:30.054+0000 7f245c871640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 40047) UID: 1000
2024-05-17T15:19:31.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.status
2024-05-17T15:19:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml
2024-05-17T15:19:31.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=120
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 120 = 120
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:31.248 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:32.249 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_18':
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220bd1b260f
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220bd1b260f
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:09 2024
2024-05-17T15:19:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:09 2024
2024-05-17T15:19:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:09 2024
2024-05-17T15:19:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: bc8b79fe-c60a-42f2-8eff-db2c0775304b
2024-05-17T15:19:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_18 snap
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_18
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:32.320 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.export
2024-05-17T15:19:32.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.export
2024-05-17T15:19:32.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.export
2024-05-17T15:19:32.321 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.export
2024-05-17T15:19:32.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:32.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.export
2024-05-17T15:19:32.842 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:32.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.export
2024-05-17T15:19:32.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_18.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_18.export
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_19
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_19 snap
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:33.480 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~189s
2024-05-17T15:19:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_19
2024-05-17T15:19:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:35.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_19 started
2024-05-17T15:19:35.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:35.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:35.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:35.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:35.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:35.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:35.545 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:35.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:35.547 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:35.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:35.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:35.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:35.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:35.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:35.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19
2024-05-17T15:19:35.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:35.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:35.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:35.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:35.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19
2024-05-17T15:19:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:35.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:35.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:35.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_19
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_19
2024-05-17T15:19:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_19.status
2024-05-17T15:19:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_19.status
2024-05-17T15:19:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:35.854 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.status
2024-05-17T15:19:35.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_19.status
2024-05-17T15:19:35.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_19.status
2024-05-17T15:19:35.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:35.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:35.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_19
2024-05-17T15:19:35.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:35.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:35.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:35.857 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19
2024-05-17T15:19:36.565 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 122
2024-05-17T15:19:36.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.status
2024-05-17T15:19:36.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:36.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.status
2024-05-17T15:19:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml
2024-05-17T15:19:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:36.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:36.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=122
2024-05-17T15:19:36.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:36.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:36.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:36.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:36.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:36.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:36.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:36.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:36.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml
2024-05-17T15:19:36.840 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:36.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=53
2024-05-17T15:19:36.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 122 = 53
2024-05-17T15:19:36.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:36.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:37.055 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0
2024-05-17T15:19:37.056 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:19:37.056 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0
2024-05-17T15:19:37.056 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon
2024-05-17T15:19:37.057 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0
2024-05-17T15:19:37.060 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started
2024-05-17T15:19:37.060 INFO:tasks.rbd_mirror_thrash:waiting for 83 secs before thrashing
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml
2024-05-17T15:19:37.313 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:37.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:37.378 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:37.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:37.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:37.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 122 = ''
2024-05-17T15:19:37.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:37.379 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml
2024-05-17T15:19:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:38.278 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:38.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:38.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:38.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:38.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 122 = ''
2024-05-17T15:19:38.279 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:38.280 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:19:38.883 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~195s
2024-05-17T15:19:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.status
2024-05-17T15:19:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml
2024-05-17T15:19:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=122
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 122 = 122
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_19':
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12204d8e573
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12204d8e573
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:24 2024
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:24 2024
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:24 2024
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 0c1ecc0c-82a3-4c9c-96a6-bc3f7117e70a
2024-05-17T15:19:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_19 snap
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_19
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.export
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.export
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.export
2024-05-17T15:19:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.export
2024-05-17T15:19:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:41.240 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.export
2024-05-17T15:19:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:41.546 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.export
2024-05-17T15:19:41.623 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_19.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_19.export
2024-05-17T15:19:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_20
2024-05-17T15:19:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_20 snap
2024-05-17T15:19:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_20
2024-05-17T15:19:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:44.287 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~200s
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_20 started
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:44.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20
2024-05-17T15:19:44.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:44.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:44.636 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20
2024-05-17T15:19:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_20']
2024-05-17T15:19:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:44.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:44.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:44.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20
2024-05-17T15:19:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_20
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_20
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_20.status
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_20.status
2024-05-17T15:19:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.status
2024-05-17T15:19:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_20.status
2024-05-17T15:19:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_20.status
2024-05-17T15:19:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_20
2024-05-17T15:19:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20
2024-05-17T15:19:45.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 124
2024-05-17T15:19:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.status
2024-05-17T15:19:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.status
2024-05-17T15:19:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml
2024-05-17T15:19:45.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:45.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=124
2024-05-17T15:19:45.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:45.727 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:45.728 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml
2024-05-17T15:19:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:46.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:46.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:46.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:46.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 124 = ''
2024-05-17T15:19:46.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:46.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml
2024-05-17T15:19:46.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:46.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 124 = ''
2024-05-17T15:19:46.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:46.484 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.status
2024-05-17T15:19:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:47.287 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml
2024-05-17T15:19:47.356 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=124
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 124 = 124
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:47.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_20':
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220be0ec095
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220be0ec095
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:33 2024
2024-05-17T15:19:48.428 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:33 2024
2024-05-17T15:19:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:33 2024
2024-05-17T15:19:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 1f368bc9-6917-4e4c-a45d-3582c17ee0f7
2024-05-17T15:19:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:48.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:48.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_20 snap
2024-05-17T15:19:48.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:48.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_20
2024-05-17T15:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.export
2024-05-17T15:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.export
2024-05-17T15:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.export
2024-05-17T15:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.export
2024-05-17T15:19:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.export
2024-05-17T15:19:49.000 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.export
2024-05-17T15:19:49.089 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_20.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_20.export
2024-05-17T15:19:49.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:49.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:49.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_21
2024-05-17T15:19:49.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_21 snap
2024-05-17T15:19:49.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:49.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:49.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:49.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:49.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:49.691 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~205s
2024-05-17T15:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_21
2024-05-17T15:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started
2024-05-17T15:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_21 started
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21
2024-05-17T15:19:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:51.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:51.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21
2024-05-17T15:19:51.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:51.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:51.710 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:51.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:51.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:51.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:51.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:51.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:51.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:51.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:51.732 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21
2024-05-17T15:19:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_21']
2024-05-17T15:19:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:51.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:51.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21
2024-05-17T15:19:51.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:51.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:51.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:51.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:52.000 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:52.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:52.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_21
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_21
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_21.status
2024-05-17T15:19:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_21.status
2024-05-17T15:19:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.status
2024-05-17T15:19:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_21.status
2024-05-17T15:19:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_21.status
2024-05-17T15:19:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_21
2024-05-17T15:19:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:52.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:52.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21
2024-05-17T15:19:52.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 126
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.status
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.status
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml
2024-05-17T15:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:52.788 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:52.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=126
2024-05-17T15:19:52.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:52.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:52.791 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:19:52.992 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:52.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:52.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:52.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:52.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:52.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml
2024-05-17T15:19:52.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 126 = ''
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:19:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml
2024-05-17T15:19:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:53.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:19:53.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:19:53.536 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:19:53.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:19:53.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 126 = ''
2024-05-17T15:19:53.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:53.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:19:54.338 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.status
2024-05-17T15:19:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml
2024-05-17T15:19:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:54.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=126
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 126 = 126
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:55.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~211s
2024-05-17T15:19:55.412 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_21':
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220c7e0469d
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220c7e0469d
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:19:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:45 2024
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:45 2024
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:45 2024
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7d839829-ed18-4ca7-9b9a-0db47f33f6f5
2024-05-17T15:19:55.476 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:19:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_21 snap
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_21
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.export
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.export
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.export
2024-05-17T15:19:55.480 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.export
2024-05-17T15:19:55.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.export
2024-05-17T15:19:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:19:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.export
2024-05-17T15:19:56.128 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_21.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_21.export
2024-05-17T15:19:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:19:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_22
2024-05-17T15:19:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_22 snap
2024-05-17T15:19:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:19:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_22
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:19:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_22 started
2024-05-17T15:19:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:19:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:19:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:19:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:19:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:19:58.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:19:58.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:19:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:19:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:19:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22
2024-05-17T15:19:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_22']
2024-05-17T15:19:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:19:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:19:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:19:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:19:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:19:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:19:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:19:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:19:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:19:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:19:58.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:58.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:58.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:19:58.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22
2024-05-17T15:19:59.047 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:19:59.048 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:19:59.048 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:19:59.049 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:19:59.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:19:59.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:19:59.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:19:59.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:19:59.051 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_22
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_22
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_22.status
2024-05-17T15:19:59.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_22.status
2024-05-17T15:19:59.053 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:59.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.status
2024-05-17T15:19:59.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_22.status
2024-05-17T15:19:59.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_22.status
2024-05-17T15:19:59.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:19:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:19:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_22
2024-05-17T15:19:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22
2024-05-17T15:19:59.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 128
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.status
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.status
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml
2024-05-17T15:19:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:19:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:19:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=128
2024-05-17T15:19:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:19:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:19:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:20:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:20:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:20:00.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml
2024-05-17T15:20:00.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:00.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:00.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 128 = ''
2024-05-17T15:20:00.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:00.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:00.500 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~216s
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml
2024-05-17T15:20:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 128 = ''
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:01.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:20:01.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:01.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:01.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:20:01.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.status
2024-05-17T15:20:01.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml
2024-05-17T15:20:01.384 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=128
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 128 = 128
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:02.054 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:20:02.048+0000 7fc15d65d640 -1 reset not still connected to 0x55fd0edcfee0
2024-05-17T15:20:02.054 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:20:02.048+0000 7fc15d65d640 -1 reset not still connected to 0x55fd121f8340
2024-05-17T15:20:02.054 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:20:02.048+0000 7fc15d65d640 -1 reset not still connected to 0x55fd123e01a0
2024-05-17T15:20:02.054 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:20:02.048+0000 7fc15d65d640 -1 reset not still connected to 0x55fd123e0340
2024-05-17T15:20:02.055 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:20:02.048+0000 7fc15d65d640 -1 reset not still connected to 0x55fd123e09c0
2024-05-17T15:20:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap
2024-05-17T15:20:02.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_22':
2024-05-17T15:20:02.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:02.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:02.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 122089d2ca1b
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.122089d2ca1b
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:11:51 2024
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:11:51 2024
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:11:51 2024
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: e1e9a751-7f3a-4c40-a3f6-7a2a2551ce54
2024-05-17T15:20:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_22 snap
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_22
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:02.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.export
2024-05-17T15:20:02.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.export
2024-05-17T15:20:02.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.export
2024-05-17T15:20:02.517 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.export
2024-05-17T15:20:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.export
2024-05-17T15:20:03.035 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:03.050 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.export
2024-05-17T15:20:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_22.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_22.export
2024-05-17T15:20:03.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:03.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:03.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_23
2024-05-17T15:20:03.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_23 snap
2024-05-17T15:20:03.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:03.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:03.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:03.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:03.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_23
2024-05-17T15:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_23 started
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23
2024-05-17T15:20:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:05.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:05.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:05.758 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:05.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:05.759 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:05.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:05.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23
2024-05-17T15:20:05.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:05.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:05.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23
2024-05-17T15:20:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_23']
2024-05-17T15:20:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:05.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:05.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:05.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:05.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:05.892 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:05.904 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~222s
2024-05-17T15:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23
2024-05-17T15:20:06.065 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:06.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:06.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:06.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:06.069 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_23
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_23
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_23.status
2024-05-17T15:20:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_23.status
2024-05-17T15:20:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.status
2024-05-17T15:20:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_23.status
2024-05-17T15:20:06.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_23.status
2024-05-17T15:20:06.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:06.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_23
2024-05-17T15:20:06.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:06.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:06.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:06.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23
2024-05-17T15:20:06.790 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 130
2024-05-17T15:20:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.status
2024-05-17T15:20:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.status
2024-05-17T15:20:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml
2024-05-17T15:20:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:06.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:06.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=130
2024-05-17T15:20:06.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:06.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:06.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml
2024-05-17T15:20:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:07.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:07.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 130 = ''
2024-05-17T15:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:07.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:07.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:07.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml
2024-05-17T15:20:07.550 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:07.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 130 = ''
2024-05-17T15:20:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:08.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.status
2024-05-17T15:20:08.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml
2024-05-17T15:20:08.422 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=130
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 130 = 130
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap
2024-05-17T15:20:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_23':
2024-05-17T15:20:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 122032dca802
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.122032dca802
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:03 2024
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:03 2024
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:03 2024
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 21cc1395-0501-4467-8166-6ae00529e254
2024-05-17T15:20:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:09.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:09.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_23 snap
2024-05-17T15:20:09.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:09.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_23
2024-05-17T15:20:09.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:09.559 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.export
2024-05-17T15:20:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.export
2024-05-17T15:20:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.export
2024-05-17T15:20:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.export
2024-05-17T15:20:09.778 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:09.785 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.export
2024-05-17T15:20:10.052 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:10.062 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.export
2024-05-17T15:20:10.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_23.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_23.export
2024-05-17T15:20:10.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:10.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:10.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_24
2024-05-17T15:20:10.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_24 snap
2024-05-17T15:20:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:11.309 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~227s
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_24
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_24 started
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24
2024-05-17T15:20:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:12.810 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:12.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:12.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:12.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:12.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:12.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:12.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:12.837 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24
2024-05-17T15:20:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_24']
2024-05-17T15:20:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:12.945 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:12.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:12.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:12.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:12.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24
2024-05-17T15:20:13.115 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:13.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:13.116 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:13.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:13.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:13.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_24
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_24
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:13.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:13.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_24.status
2024-05-17T15:20:13.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_24.status
2024-05-17T15:20:13.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:13.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.status
2024-05-17T15:20:13.122 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_24.status
2024-05-17T15:20:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_24.status
2024-05-17T15:20:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_24
2024-05-17T15:20:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:13.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:13.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:13.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24
2024-05-17T15:20:13.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 132
2024-05-17T15:20:13.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.status
2024-05-17T15:20:13.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.status
2024-05-17T15:20:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml
2024-05-17T15:20:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:13.891 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=132
2024-05-17T15:20:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:14.068 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0
2024-05-17T15:20:14.069 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15
2024-05-17T15:20:14.069 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons
2024-05-17T15:20:14.069 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0
2024-05-17T15:20:14.069 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:20:14.070 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T15:20:14.065+0000 7ffb09515640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0  (PID: 39403) UID: 1000
2024-05-17T15:20:14.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:14.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:14.095 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:14.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:14.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:14.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml
2024-05-17T15:20:14.096 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:14.166 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:14.167 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:14.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:14.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:14.169 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 132 = ''
2024-05-17T15:20:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:14.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml
2024-05-17T15:20:14.572 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 132 = ''
2024-05-17T15:20:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:14.646 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:15.272 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:20:15.265+0000 7f905860c640 -1 reset not still connected to 0x55d26a0a5380
2024-05-17T15:20:15.274 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:20:15.269+0000 7f905860c640 -1 reset not still connected to 0x55d26dcff790
2024-05-17T15:20:15.274 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:20:15.269+0000 7f905860c640 -1 reset not still connected to 0x55d26deb56c0
2024-05-17T15:20:15.274 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:20:15.269+0000 7f905860c640 -1 reset not still connected to 0x55d26fd0dad0
2024-05-17T15:20:15.277 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:20:15.273+0000 7f905860c640 -1 reset not still connected to 0x55d273413ad0
2024-05-17T15:20:15.283 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:20:15.281+0000 7f905860c640 -1 reset not still connected to 0x55d273413ad0
2024-05-17T15:20:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:15.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:15.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.status
2024-05-17T15:20:15.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml
2024-05-17T15:20:15.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:15.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=132
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 132 = 132
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap
2024-05-17T15:20:16.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_24':
2024-05-17T15:20:16.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220f67780da
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220f67780da
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:11 2024
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:11 2024
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:11 2024
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7f23dbe5-352e-45f0-ba06-9261e41a7df4
2024-05-17T15:20:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_24 snap
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_24
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.export
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.export
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.export
2024-05-17T15:20:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.export
2024-05-17T15:20:16.713 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~232s
2024-05-17T15:20:16.811 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:16.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.export
2024-05-17T15:20:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:17.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.export
2024-05-17T15:20:17.274 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_24.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_24.export
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_25
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_25 snap
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:17.308 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_25
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:18.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:18.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:18.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_25 started
2024-05-17T15:20:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25
2024-05-17T15:20:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:19.834 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:19.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:19.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:19.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25
2024-05-17T15:20:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_25']
2024-05-17T15:20:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25
2024-05-17T15:20:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25
2024-05-17T15:20:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:20.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:20.142 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:20.143 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:20.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:20.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:20.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:20.146 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_25
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_25
2024-05-17T15:20:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_25.status
2024-05-17T15:20:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_25.status
2024-05-17T15:20:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.status
2024-05-17T15:20:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_25.status
2024-05-17T15:20:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_25.status
2024-05-17T15:20:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:20.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_25
2024-05-17T15:20:20.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:20.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:20.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:20.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25
2024-05-17T15:20:20.171 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0
2024-05-17T15:20:20.172 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon
2024-05-17T15:20:20.172 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0
2024-05-17T15:20:20.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started
2024-05-17T15:20:20.174 INFO:tasks.rbd_mirror_thrash:waiting for 67 secs before thrashing
2024-05-17T15:20:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 134
2024-05-17T15:20:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.status
2024-05-17T15:20:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.status
2024-05-17T15:20:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml
2024-05-17T15:20:20.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:20.925 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=134
2024-05-17T15:20:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:21.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:21.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:21.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:21.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:21.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml
2024-05-17T15:20:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:21.204 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=59
2024-05-17T15:20:21.205 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 134 = 59
2024-05-17T15:20:21.205 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:21.205 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:21.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml
2024-05-17T15:20:21.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:21.683 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=59
2024-05-17T15:20:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 134 = 59
2024-05-17T15:20:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:21.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:22.118 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~238s
2024-05-17T15:20:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml
2024-05-17T15:20:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=59
2024-05-17T15:20:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 134 = 59
2024-05-17T15:20:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:20:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.status
2024-05-17T15:20:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml
2024-05-17T15:20:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:24.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:24.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=134
2024-05-17T15:20:24.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 134 = 134
2024-05-17T15:20:24.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:25.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_25':
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12203e5f250c
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12203e5f250c
2024-05-17T15:20:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:18 2024
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:18 2024
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:18 2024
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 256ee516-bc3e-401b-8783-9eec22739431
2024-05-17T15:20:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_25 snap
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_25
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.export
2024-05-17T15:20:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.export
2024-05-17T15:20:25.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.export
2024-05-17T15:20:25.303 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.export
2024-05-17T15:20:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:25.521 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.export
2024-05-17T15:20:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:25.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.export
2024-05-17T15:20:25.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_25.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_25.export
2024-05-17T15:20:25.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_26
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_26 snap
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:27.522 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~243s
2024-05-17T15:20:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_26
2024-05-17T15:20:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_26 started
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26
2024-05-17T15:20:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:28.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:28.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26
2024-05-17T15:20:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_26']
2024-05-17T15:20:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:28.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:28.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26
2024-05-17T15:20:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:28.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:28.903 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_26
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_26
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_26.status
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_26.status
2024-05-17T15:20:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:28.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.status
2024-05-17T15:20:28.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_26.status
2024-05-17T15:20:28.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_26.status
2024-05-17T15:20:28.907 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_26
2024-05-17T15:20:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26
2024-05-17T15:20:29.584 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 136
2024-05-17T15:20:29.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.status
2024-05-17T15:20:29.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:29.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:29.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:29.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.status
2024-05-17T15:20:29.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml
2024-05-17T15:20:29.601 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:29.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:29.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=136
2024-05-17T15:20:29.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:29.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:29.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:29.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:29.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:29.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:29.860 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:29.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:29.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml
2024-05-17T15:20:29.861 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:29.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:29.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=60
2024-05-17T15:20:29.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 136 = 60
2024-05-17T15:20:29.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:29.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:30.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:30.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:30.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:30.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml
2024-05-17T15:20:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:30.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:30.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:30.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:30.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:30.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:30.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:30.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 136 = ''
2024-05-17T15:20:30.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:30.404 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.status
2024-05-17T15:20:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml
2024-05-17T15:20:31.207 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:31.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:31.348 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=136
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 136 = 136
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:31.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:32.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_26':
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12206f494bbe
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12206f494bbe
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:24 2024
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:24 2024
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:24 2024
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 798e27a4-f2bd-44ea-ac4a-5295e0c6ebb8
2024-05-17T15:20:32.411 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_26 snap
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_26
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.export
2024-05-17T15:20:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.export
2024-05-17T15:20:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.export
2024-05-17T15:20:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.export
2024-05-17T15:20:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:32.637 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.export
2024-05-17T15:20:32.925 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~249s
2024-05-17T15:20:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.export
2024-05-17T15:20:33.029 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_26.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_26.export
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_27
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_27 snap
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_27
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:34.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:34.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:34.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:35.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_27 started
2024-05-17T15:20:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:35.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:35.656 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:35.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:35.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:35.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27
2024-05-17T15:20:35.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_27']
2024-05-17T15:20:35.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:35.792 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:35.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:35.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:35.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:35.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:35.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:35.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:35.798 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:35.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:35.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:35.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:35.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27
2024-05-17T15:20:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_27
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_27
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_27.status
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_27.status
2024-05-17T15:20:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.status
2024-05-17T15:20:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_27.status
2024-05-17T15:20:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_27.status
2024-05-17T15:20:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:35.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:35.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_27
2024-05-17T15:20:35.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:35.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:35.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:35.970 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27
2024-05-17T15:20:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 138
2024-05-17T15:20:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.status
2024-05-17T15:20:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.status
2024-05-17T15:20:36.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml
2024-05-17T15:20:36.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:36.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=138
2024-05-17T15:20:36.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:36.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:36.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml
2024-05-17T15:20:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 138 = ''
2024-05-17T15:20:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml
2024-05-17T15:20:37.421 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:37.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:37.490 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 138 = ''
2024-05-17T15:20:37.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:37.494 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:38.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:38.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:38.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:38.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml
2024-05-17T15:20:38.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:38.329 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~254s
2024-05-17T15:20:38.363 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:38.363 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:38.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:38.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:38.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:38.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:38.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 138 = ''
2024-05-17T15:20:38.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:38.367 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:20:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.status
2024-05-17T15:20:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml
2024-05-17T15:20:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:40.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=138
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 138 = 138
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:41.042 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap
2024-05-17T15:20:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_27':
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220807cd83
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220807cd83
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:32 2024
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:32 2024
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:32 2024
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 88d5ad35-b4ea-4cf0-968b-4a1d374fed30
2024-05-17T15:20:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_27 snap
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_27
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.export
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.export
2024-05-17T15:20:41.109 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.export
2024-05-17T15:20:41.110 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.export
2024-05-17T15:20:41.318 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.export
2024-05-17T15:20:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:41.627 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.export
2024-05-17T15:20:41.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_27.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_27.export
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_28
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_28 snap
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:43.732 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~259s
2024-05-17T15:20:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_28
2024-05-17T15:20:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:43.743 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_28 started
2024-05-17T15:20:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:44.744 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28
2024-05-17T15:20:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28
2024-05-17T15:20:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_28']
2024-05-17T15:20:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28
2024-05-17T15:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:44.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:44.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:44.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:44.906 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28
2024-05-17T15:20:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_28
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_28
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_28.status
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_28.status
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.status
2024-05-17T15:20:45.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_28.status
2024-05-17T15:20:45.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:45.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_28.status
2024-05-17T15:20:45.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:45.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_28
2024-05-17T15:20:45.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:45.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:45.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:45.076 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28
2024-05-17T15:20:45.761 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 140
2024-05-17T15:20:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.status
2024-05-17T15:20:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:45.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.status
2024-05-17T15:20:45.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml
2024-05-17T15:20:45.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:45.832 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:45.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=140
2024-05-17T15:20:45.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:45.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:45.836 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:46.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:46.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:46.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:46.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:46.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:46.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml
2024-05-17T15:20:46.038 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:46.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:46.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:46.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 140 = ''
2024-05-17T15:20:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:46.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml
2024-05-17T15:20:46.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:46.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:46.577 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:46.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:46.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:46.580 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:46.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:46.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 140 = ''
2024-05-17T15:20:46.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:46.581 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.status
2024-05-17T15:20:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml
2024-05-17T15:20:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:47.444 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=140
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 140 = 140
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:47.448 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:47.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:47.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:48.449 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap
2024-05-17T15:20:48.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_28':
2024-05-17T15:20:48.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:48.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:48.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:48.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220e3c921e1
2024-05-17T15:20:48.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220e3c921e1
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:46 2024
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:46 2024
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:46 2024
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 3fef99bd-821a-4e44-b224-1260889e1805
2024-05-17T15:20:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_28 snap
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_28
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.export
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.export
2024-05-17T15:20:48.515 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.export
2024-05-17T15:20:48.516 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.export
2024-05-17T15:20:48.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:48.726 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.export
2024-05-17T15:20:49.058 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:49.070 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.export
2024-05-17T15:20:49.137 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~265s
2024-05-17T15:20:49.147 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_28.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_28.export
2024-05-17T15:20:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_29
2024-05-17T15:20:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_29 snap
2024-05-17T15:20:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:49.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:49.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:49.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:49.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_29
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:50.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_29 started
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:51.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:51.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:51.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:51.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:51.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:51.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:51.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:51.823 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:51.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:51.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:51.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:51.843 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29
2024-05-17T15:20:51.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_29']
2024-05-17T15:20:51.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:51.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:51.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:51.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:51.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:51.976 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:51.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:51.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:51.977 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29
2024-05-17T15:20:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:52.133 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:52.133 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:52.135 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_29
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_29
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:52.137 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_29.status
2024-05-17T15:20:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_29.status
2024-05-17T15:20:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.status
2024-05-17T15:20:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_29.status
2024-05-17T15:20:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_29.status
2024-05-17T15:20:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_29
2024-05-17T15:20:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29
2024-05-17T15:20:52.841 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 142
2024-05-17T15:20:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.status
2024-05-17T15:20:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.status
2024-05-17T15:20:52.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml
2024-05-17T15:20:52.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:52.919 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:52.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=142
2024-05-17T15:20:52.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:52.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:52.922 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:53.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:53.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:53.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:53.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:53.124 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:53.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml
2024-05-17T15:20:53.125 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 142 = ''
2024-05-17T15:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:20:53.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:53.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:53.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:53.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:53.598 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml
2024-05-17T15:20:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:53.665 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:53.666 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:20:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:20:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:20:53.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:20:53.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 142 = ''
2024-05-17T15:20:53.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:53.669 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:20:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.status
2024-05-17T15:20:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml
2024-05-17T15:20:54.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:54.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:20:54.541 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~270s
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=142
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 142 = 142
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:54.542 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:54.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:54.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:20:54.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:55.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap
2024-05-17T15:20:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_29':
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12205da98c6d
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12205da98c6d
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:12:52 2024
2024-05-17T15:20:55.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:12:52 2024
2024-05-17T15:20:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:12:52 2024
2024-05-17T15:20:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:20:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:20:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:20:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 7fda8009-e76c-42ba-8a4f-b0fc43ecf4b6
2024-05-17T15:20:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:20:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_29 snap
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_29
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.export
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.export
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.export
2024-05-17T15:20:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.export
2024-05-17T15:20:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.export
2024-05-17T15:20:56.149 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:20:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.export
2024-05-17T15:20:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_29.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_29.export
2024-05-17T15:20:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:20:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_30
2024-05-17T15:20:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_30 snap
2024-05-17T15:20:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:56.283 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:56.283 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:56.283 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:20:56.283 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:20:57.864 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_30
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:57.865 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:20:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_30 started
2024-05-17T15:20:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:20:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:20:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:20:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:20:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:20:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:20:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:20:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:20:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:20:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:20:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:20:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:58.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:20:58.897 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30
2024-05-17T15:20:59.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_30']
2024-05-17T15:20:59.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:20:59.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:20:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:20:59.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:20:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:20:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:20:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30
2024-05-17T15:20:59.185 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:20:59.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:20:59.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:20:59.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:20:59.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_30
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_30
2024-05-17T15:20:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:20:59.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:59.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:59.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:59.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_30.status
2024-05-17T15:20:59.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_30.status
2024-05-17T15:20:59.190 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.status
2024-05-17T15:20:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_30.status
2024-05-17T15:20:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_30.status
2024-05-17T15:20:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:20:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:20:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_30
2024-05-17T15:20:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30
2024-05-17T15:20:59.856 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 144
2024-05-17T15:20:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.status
2024-05-17T15:20:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:20:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:20:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:20:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.status
2024-05-17T15:20:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml
2024-05-17T15:20:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:20:59.926 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:20:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=144
2024-05-17T15:20:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:20:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:20:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:20:59.943 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~276s
2024-05-17T15:21:00.061 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8628920656205614) > weight (0.25)
2024-05-17T15:21:00.061 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5853231698175375) > weight (0.25)
2024-05-17T15:21:00.061 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5560544006764336) > weight (0.25)
2024-05-17T15:21:00.062 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.33683597700670587) > weight (0.25)
2024-05-17T15:21:00.062 INFO:tasks.rbd_mirror_thrash:waiting for 110 secs before thrashing
2024-05-17T15:21:00.130 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:21:00.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:00.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:00.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:21:00.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:21:00.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml
2024-05-17T15:21:00.131 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:00.194 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:00.195 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 144 = ''
2024-05-17T15:21:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:21:00.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:21:00.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:00.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:00.599 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:21:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:21:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml
2024-05-17T15:21:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:00.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:00.668 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:00.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:00.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:00.671 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:00.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:00.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 144 = ''
2024-05-17T15:21:00.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:00.672 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:21:01.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:21:01.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:01.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:01.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:21:01.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.status
2024-05-17T15:21:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml
2024-05-17T15:21:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=144
2024-05-17T15:21:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 144 = 144
2024-05-17T15:21:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap
2024-05-17T15:21:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:21:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:21:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap
2024-05-17T15:21:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_30':
2024-05-17T15:21:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:21:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:21:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:21:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 12209706b187
2024-05-17T15:21:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.12209706b187
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:02 2024
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:02 2024
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:02 2024
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: b890af64-5f4d-4763-9ed5-689fae1e3fdc
2024-05-17T15:21:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_30 snap
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_30
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:21:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.export
2024-05-17T15:21:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.export
2024-05-17T15:21:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.export
2024-05-17T15:21:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.export
2024-05-17T15:21:02.818 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.export
2024-05-17T15:21:03.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:03.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.export
2024-05-17T15:21:03.201 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_30.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_30.export
2024-05-17T15:21:03.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:03.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:21:03.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_31
2024-05-17T15:21:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_31 snap
2024-05-17T15:21:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:21:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_31
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:21:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:05.347 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~281s
2024-05-17T15:21:05.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_31 started
2024-05-17T15:21:05.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:05.869 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:05.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:05.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:21:05.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:21:05.870 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:21:05.871 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:21:05.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:21:05.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:21:05.872 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:21:05.873 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:21:05.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:21:05.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31
2024-05-17T15:21:05.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:21:05.876 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:21:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:21:05.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:21:05.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:21:05.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:21:05.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:21:05.879 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:21:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31
2024-05-17T15:21:06.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_31']
2024-05-17T15:21:06.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:21:06.001 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:21:06.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:21:06.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:21:06.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:21:06.005 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:21:06.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:21:06.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:21:06.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:21:06.024 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:06.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:06.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:06.025 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31
2024-05-17T15:21:06.176 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:21:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:21:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:21:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:21:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:21:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:21:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:21:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_31
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_31
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:06.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_31.status
2024-05-17T15:21:06.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_31.status
2024-05-17T15:21:06.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:21:06.183 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.status
2024-05-17T15:21:06.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_31.status
2024-05-17T15:21:06.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_31.status
2024-05-17T15:21:06.184 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:21:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_31
2024-05-17T15:21:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31
2024-05-17T15:21:06.884 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 146
2024-05-17T15:21:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.status
2024-05-17T15:21:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.status
2024-05-17T15:21:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml
2024-05-17T15:21:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:21:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=146
2024-05-17T15:21:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:21:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:07.163 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml
2024-05-17T15:21:07.163 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:07.233 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:07.236 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=65
2024-05-17T15:21:07.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 146 = 65
2024-05-17T15:21:07.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:07.237 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:21:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml
2024-05-17T15:21:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:07.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 146 = ''
2024-05-17T15:21:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:21:08.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:08.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:08.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:08.509 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:08.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.status
2024-05-17T15:21:08.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml
2024-05-17T15:21:08.510 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:08.582 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:08.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=146
2024-05-17T15:21:08.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 146 = 146
2024-05-17T15:21:08.585 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:21:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:09.586 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap
2024-05-17T15:21:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_31':
2024-05-17T15:21:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:21:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220b054e187
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220b054e187
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:14 2024
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:14 2024
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:14 2024
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:21:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:21:09.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: 2a498d86-9a3f-4036-b490-785be695571e
2024-05-17T15:21:09.655 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:21:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_31 snap
2024-05-17T15:21:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_31
2024-05-17T15:21:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:21:09.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.export
2024-05-17T15:21:09.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.export
2024-05-17T15:21:09.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.export
2024-05-17T15:21:09.659 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.export
2024-05-17T15:21:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.export
2024-05-17T15:21:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:10.193 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.export
2024-05-17T15:21:10.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_31.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_31.export
2024-05-17T15:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_32
2024-05-17T15:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_32 snap
2024-05-17T15:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:21:10.751 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~286s
2024-05-17T15:21:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_32
2024-05-17T15:21:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:21:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:12.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_32 started
2024-05-17T15:21:12.613 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:12.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:12.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:12.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:21:12.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:21:12.614 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:21:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:21:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:21:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:21:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:21:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:21:12.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:21:12.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32
2024-05-17T15:21:12.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:21:12.620 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:21:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:21:12.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:12.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:12.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:12.644 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32
2024-05-17T15:21:12.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_32']
2024-05-17T15:21:12.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:21:12.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:21:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:21:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:21:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:21:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:21:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:21:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:21:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32
2024-05-17T15:21:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:21:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:21:12.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:21:12.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:21:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:21:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:21:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:21:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_32
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_32
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_32.status
2024-05-17T15:21:12.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_32.status
2024-05-17T15:21:12.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:21:12.934 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.status
2024-05-17T15:21:12.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_32.status
2024-05-17T15:21:12.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_32.status
2024-05-17T15:21:12.935 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:21:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_32
2024-05-17T15:21:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32
2024-05-17T15:21:13.617 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 148
2024-05-17T15:21:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.status
2024-05-17T15:21:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.status
2024-05-17T15:21:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml
2024-05-17T15:21:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:13.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:21:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=148
2024-05-17T15:21:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:21:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml
2024-05-17T15:21:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:13.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:13.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=66
2024-05-17T15:21:13.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 148 = 66
2024-05-17T15:21:13.963 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:13.964 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:21:14.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:14.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:14.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:14.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:14.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:14.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml
2024-05-17T15:21:14.366 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:14.437 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:14.438 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:14.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:14.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:14.440 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:14.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:14.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 148 = ''
2024-05-17T15:21:14.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:14.441 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:21:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:15.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml
2024-05-17T15:21:15.243 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 148 = ''
2024-05-17T15:21:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1.6
2024-05-17T15:21:16.154 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~292s
2024-05-17T15:21:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:16.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.status
2024-05-17T15:21:16.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml
2024-05-17T15:21:16.912 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:16.978 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=148
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 148 = 148
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:21:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap
2024-05-17T15:21:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_32':
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220995c0b50
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220995c0b50
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:24 2024
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:24 2024
2024-05-17T15:21:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:24 2024
2024-05-17T15:21:18.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:21:18.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:21:18.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:21:18.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: ebb6c97f-7a9a-4158-a875-14c29371e6d8
2024-05-17T15:21:18.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:21:18.060 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_32 snap
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_32
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.export
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.export
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.export
2024-05-17T15:21:18.061 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.export
2024-05-17T15:21:18.263 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:18.270 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.export
2024-05-17T15:21:18.593 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:18.607 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.export
2024-05-17T15:21:18.683 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_32.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_32.export
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_33
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_33 snap
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_started cluster1 mirror image_33
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local state=started
2024-05-17T15:21:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:20.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:21:20.690 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:21.558 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~297s
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test_image_replay_state cluster1 mirror image_33 started
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local test_state=started
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_result
2024-05-17T15:21:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local current_state=stopped
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ grep -i state
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_instance=cluster1
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance=cluster1
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local loop_instance
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16
2024-05-17T15:21:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sleep 0
2024-05-17T15:21:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' cluster1 '!=' cluster1 ']'
2024-05-17T15:21:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ seq 0 3
2024-05-17T15:21:21.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:21:21.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33
2024-05-17T15:21:21.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:0
2024-05-17T15:21:21.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:21.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:21:21.696 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:0 cluster instance
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:0
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=0
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 0 = cluster1:0 ']'
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=0
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:0 cluster1
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:0
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:0
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=0
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 0 = cluster1:0 ']'
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=0
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=0
2024-05-17T15:21:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket'
2024-05-17T15:21:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok
2024-05-17T15:21:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33
2024-05-17T15:21:21.824 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Invalid command: unused arguments: ['mirror/image_33']
2024-05-17T15:21:21.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd mirror status :  get status for rbd mirror
2024-05-17T15:21:21.825 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:admin_socket: invalid command
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster=cluster1:1
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ shift
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set_cluster_instance cluster1:1 cluster instance
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local val=cluster1:1
2024-05-17T15:21:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster_var_name=cluster
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local instance_var_name=instance
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cluster=cluster1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ instance=1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 1 = cluster1:1 ']'
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval cluster=cluster1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ eval instance=1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ daemon_asok_file cluster1:1 cluster1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local local_cluster=cluster1:1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster=cluster1
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance
2024-05-17T15:21:21.829 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local val=cluster1:1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local cluster_var_name=local_cluster
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ local instance_var_name=instance
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ cluster=cluster1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ instance=1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ '[' 1 = cluster1:1 ']'
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval local_cluster=cluster1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ local_cluster=cluster1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ eval instance=1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ instance=1
2024-05-17T15:21:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket'
2024-05-17T15:21:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok
2024-05-17T15:21:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33
2024-05-17T15:21:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ return 0
2024-05-17T15:21:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ status_result='    "state": "Replaying"'
2024-05-17T15:21:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ echo '    "state": "Replaying"'
2024-05-17T15:21:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ grep -i Replaying
2024-05-17T15:21:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:    "state": "Replaying"
2024-05-17T15:21:22.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ current_state=started
2024-05-17T15:21:22.007 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test started = started
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_33
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = journal ']'
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ '[' snapshot = snapshot ']'
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_33
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_cluster=cluster1
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster2-mirror-image_33.status
2024-05-17T15:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster2-mirror-image_33.status
2024-05-17T15:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local status_log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.status
2024-05-17T15:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ mkfname cluster1-mirror-image_33.status
2024-05-17T15:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo cluster1-mirror-image_33.status
2024-05-17T15:21:22.010 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ sed -e 's|[/ ]|_|g'
2024-05-17T15:21:22.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local local_status_log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:22.011 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ mirror_image_snapshot cluster2 mirror image_33
2024-05-17T15:21:22.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:22.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:22.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:22.012 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33
2024-05-17T15:21:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:Snapshot ID: 150
2024-05-17T15:21:22.736 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.status
2024-05-17T15:21:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.status
2024-05-17T15:21:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml
2024-05-17T15:21:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:22.793 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/id
2024-05-17T15:21:22.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snapshot_id=150
2024-05-17T15:21:22.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:22.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:22.797 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.2
2024-05-17T15:21:22.998 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml
2024-05-17T15:21:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:23.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:23.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 150 = ''
2024-05-17T15:21:23.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:23.068 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.4
2024-05-17T15:21:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml
2024-05-17T15:21:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:23.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ true
2024-05-17T15:21:23.540 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:-:1.1: Document is empty
2024-05-17T15:21:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
2024-05-17T15:21:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:^
2024-05-17T15:21:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=
2024-05-17T15:21:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 150 = ''
2024-05-17T15:21:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32
2024-05-17T15:21:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 0.8
2024-05-17T15:21:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local log=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.status
2024-05-17T15:21:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml
2024-05-17T15:21:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]'
2024-05-17T15:21:24.403 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id
2024-05-17T15:21:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local primary_snapshot_id=150
2024-05-17T15:21:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ test 150 = 150
2024-05-17T15:21:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap
2024-05-17T15:21:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster1
2024-05-17T15:21:24.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:24.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:24.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:24.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local s
2024-05-17T15:21:24.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32
2024-05-17T15:21:24.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ sleep 1
2024-05-17T15:21:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:rbd image 'image_33':
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	size 128 MiB in 32 objects
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	order 22 (4 MiB objects)
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	snapshot_count: 2
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	id: 1220ec0fc027
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	block_name_prefix: rbd_data.1220ec0fc027
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	format: 2
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	features: layering, exclusive-lock, non-primary
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	op_features:
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	flags:
2024-05-17T15:21:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	create_timestamp: Fri May 17 15:13:32 2024
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	access_timestamp: Fri May 17 15:13:32 2024
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	modify_timestamp: Fri May 17 15:13:32 2024
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	protected: False
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring state: enabled
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring mode: snapshot
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring global id: b26f73bc-22d7-4d4a-b487-999e05b95f20
2024-05-17T15:21:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:	mirroring primary: false
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ compare_image_snaps mirror image_33 snap
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_33
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local ret=0
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local rmt_export=/tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.export
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local loc_export=/tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.export
2024-05-17T15:21:25.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.export
2024-05-17T15:21:25.478 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.export
2024-05-17T15:21:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.export
2024-05-17T15:21:26.017 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:
Exporting image: 3% complete...
Exporting image: 6% complete...
Exporting image: 9% complete...
Exporting image: 12% complete...
Exporting image: 15% complete...
Exporting image: 18% complete...
Exporting image: 21% complete...
Exporting image: 25% complete...
Exporting image: 28% complete...
Exporting image: 31% complete...
Exporting image: 34% complete...
Exporting image: 37% complete...
Exporting image: 40% complete...
Exporting image: 43% complete...
Exporting image: 46% complete...
Exporting image: 50% complete...
Exporting image: 53% complete...
Exporting image: 56% complete...
Exporting image: 59% complete...
Exporting image: 62% complete...
Exporting image: 65% complete...
Exporting image: 68% complete...
Exporting image: 71% complete...
Exporting image: 75% complete...
Exporting image: 78% complete...
Exporting image: 81% complete...
Exporting image: 84% complete...
Exporting image: 87% complete...
Exporting image: 90% complete...
Exporting image: 93% complete...
Exporting image: 96% complete...
Exporting image: 100% complete...done.
2024-05-17T15:21:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ cmp /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.export
2024-05-17T15:21:26.107 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rm -f /tmp/tmp.mfaeqiv6jj/cluster2-mirror-image_33.export /tmp/tmp.mfaeqiv6jj/cluster1-mirror-image_33.export
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ return 0
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ for i in `seq 1 ${IMAGE_COUNT}`
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ image=image_34
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ create_snap cluster2 mirror image_34 snap
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local cluster=cluster2
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local pool=mirror
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local image=image_34
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ local snap_name=snap
2024-05-17T15:21:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20
2024-05-17T15:21:26.962 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~303s
2024-05-17T15:21:26.962 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons
2024-05-17T15:21:27.174 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.577904354832307) > weight (0.25)
2024-05-17T15:21:27.175 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9098697974718256) > weight (0.25)
2024-05-17T15:21:27.175 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2
2024-05-17T15:21:27.175 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15
2024-05-17T15:21:27.176 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons
2024-05-17T15:21:27.177 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:2024-05-17T15:21:27.176+0000 7efe217ea640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2  (PID: 42679) UID: 1000
2024-05-17T15:21:27.366 INFO:tasks.ceph.cluster2.osd.1:Sent signal 15
2024-05-17T15:21:27.366 INFO:tasks.ceph.cluster2.osd.2:Sent signal 15
2024-05-17T15:21:27.366 INFO:tasks.ceph.cluster2.mon.a:Sent signal 15
2024-05-17T15:21:27.366 INFO:tasks.ceph.cluster2.mgr.x:Sent signal 15
2024-05-17T15:21:27.367 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f9065bba640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-osd -f --cluster cluster2 -i 1  (PID: 24227) UID: 0
2024-05-17T15:21:27.367 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f9065bba640 -1 osd.1 267 *** Got signal Terminated ***
2024-05-17T15:21:27.367 INFO:tasks.ceph.cluster2.osd.1.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f9065bba640 -1 osd.1 267 *** Immediate shutdown (osd_fast_shutdown=true) ***
2024-05-17T15:21:27.367 INFO:tasks.ceph.cluster2.mon.a.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f8470924640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-mon -f --cluster cluster2 -i a  (PID: 24077) UID: 0
2024-05-17T15:21:27.368 INFO:tasks.ceph.cluster2.mon.a.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f8470924640 -1 mon.a@0(leader) e1 *** Got Signal Terminated ***
2024-05-17T15:21:27.368 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f28c4f90640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-osd -f --cluster cluster2 -i 2  (PID: 24226) UID: 0
2024-05-17T15:21:27.368 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f28c4f90640 -1 osd.2 267 *** Got signal Terminated ***
2024-05-17T15:21:27.368 INFO:tasks.ceph.cluster2.osd.2.smithi171.stderr:2024-05-17T15:21:27.368+0000 7f28c4f90640 -1 osd.2 267 *** Immediate shutdown (osd_fast_shutdown=true) ***
2024-05-17T15:21:27.368 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T15:21:27.369+0000 7f62771c0640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-mgr -f --cluster cluster2 -i x  (PID: 28634) UID: 0
2024-05-17T15:21:27.368 INFO:tasks.ceph.cluster2.mgr.x.smithi072.stderr:2024-05-17T15:21:27.369+0000 7f62771c0640 -1 mgr handle_mgr_signal  *** Got signal Terminated ***
2024-05-17T15:21:31.177 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2
2024-05-17T15:21:31.177 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:21:31.178 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2
2024-05-17T15:21:31.178 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon
2024-05-17T15:21:31.178 DEBUG:teuthology.orchestra.run.smithi171:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2
2024-05-17T15:21:31.181 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started
2024-05-17T15:21:31.181 INFO:tasks.rbd_mirror_thrash:waiting for 110 secs before thrashing
2024-05-17T15:22:50.063 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0
2024-05-17T15:22:50.064 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15
2024-05-17T15:22:50.064 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons
2024-05-17T15:22:50.065 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T15:22:50.066+0000 7f2ae1ea8640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 46959) UID: 1000
2024-05-17T15:22:58.065 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0
2024-05-17T15:22:58.065 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:23:21.182 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5201751526228078) > weight (0.25)
2024-05-17T15:23:21.182 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6318123326733857) > weight (0.25)
2024-05-17T15:23:21.182 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.26041111758951585) > weight (0.25)
2024-05-17T15:23:21.183 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.2899734046438446) > weight (0.25)
2024-05-17T15:23:21.183 INFO:tasks.rbd_mirror_thrash:waiting for 78 secs before thrashing
2024-05-17T15:24:39.184 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0
2024-05-17T15:24:39.186 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15
2024-05-17T15:24:39.186 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons
2024-05-17T15:24:39.188 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T15:24:39.187+0000 7f7540fdf640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0  (PID: 48335) UID: 1000
2024-05-17T15:24:48.187 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0
2024-05-17T15:24:48.187 INFO:teuthology.orchestra.run:waiting for 600
2024-05-17T15:26:31.252 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T15:26:31.265 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi171.stderr:daemon-helper: command failed with exit status 1
2024-05-17T15:33:02.205 INFO:tasks.rbd_mirror_thrash:Failed to stop cluster1.client.mirror.0
2024-05-17T15:33:02.206 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 6
2024-05-17T15:33:02.206 ERROR:tasks.rbd_mirror_thrash:exception:
Traceback (most recent call last):
  File "/home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa/tasks/rbd_mirror_thrash.py", line 84, in _run
    self.do_thrash()
  File "/home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa/tasks/rbd_mirror_thrash.py", line 156, in do_thrash
    run.wait([daemon.proc], timeout=600)
  File "/home/teuthworker/src/git.ceph.com_teuthology_cad7edd8d899a7c59a48f7da3294fca4cb778f64/teuthology/orchestra/run.py", line 473, in wait
    check_time()
  File "/home/teuthworker/src/git.ceph.com_teuthology_cad7edd8d899a7c59a48f7da3294fca4cb778f64/teuthology/contextutil.py", line 134, in __call__
    raise MaxWhileTries(error_msg)
teuthology.exceptions.MaxWhileTries: reached maximum tries (101) after waiting for 600 seconds
2024-05-17T15:33:02.208 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:*** Caught signal (Aborted) **
2024-05-17T15:33:02.208 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: in thread 7f2ae4eed880 thread_name:rbd-mirror
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f2ae5d80520]
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f2ae5dcf197]
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 4: rbd-mirror(+0x3726ab) [0x55807a87a6ab]
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::shut_down()+0x103) [0x55807a883b73]
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::~PoolReplayer()+0x10) [0x55807a883f10]
2024-05-17T15:33:02.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 7: rbd-mirror(+0x944a08) [0x55807ae4ca08]
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 8: rbd-mirror(+0x9449f0) [0x55807ae4c9f0]
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 9: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55807a87489a]
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 10: main()
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 11: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f2ae5d67d90]
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 12: __libc_start_main()
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 13: _start()
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:2024-05-17T15:33:02.205+0000 7f2ae4eed880 -1 *** Caught signal (Aborted) **
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: in thread 7f2ae4eed880 thread_name:rbd-mirror
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f2ae5d80520]
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f2ae5dcf197]
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:33:02.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 4: rbd-mirror(+0x3726ab) [0x55807a87a6ab]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::shut_down()+0x103) [0x55807a883b73]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::~PoolReplayer()+0x10) [0x55807a883f10]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 7: rbd-mirror(+0x944a08) [0x55807ae4ca08]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 8: rbd-mirror(+0x9449f0) [0x55807ae4c9f0]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 9: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55807a87489a]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 10: main()
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 11: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f2ae5d67d90]
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 12: __libc_start_main()
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 13: _start()
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:33:02.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:
2024-05-17T15:33:02.222 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: -2777> 2024-05-17T15:22:50.066+0000 7f2ae1ea8640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 46959) UID: 1000
2024-05-17T15:33:02.231 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:     0> 2024-05-17T15:33:02.205+0000 7f2ae4eed880 -1 *** Caught signal (Aborted) **
2024-05-17T15:33:02.231 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: in thread 7f2ae4eed880 thread_name:rbd-mirror
2024-05-17T15:33:02.231 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:
2024-05-17T15:33:02.231 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:33:02.231 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f2ae5d80520]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f2ae5dcf197]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 4: rbd-mirror(+0x3726ab) [0x55807a87a6ab]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::shut_down()+0x103) [0x55807a883b73]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::~PoolReplayer()+0x10) [0x55807a883f10]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 7: rbd-mirror(+0x944a08) [0x55807ae4ca08]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 8: rbd-mirror(+0x9449f0) [0x55807ae4c9f0]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 9: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55807a87489a]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 10: main()
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 11: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f2ae5d67d90]
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 12: __libc_start_main()
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 13: _start()
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:33:02.232 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: -4846> 2024-05-17T15:22:50.066+0000 7f2ae1ea8640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0  (PID: 46959) UID: 1000
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: -4845> 2024-05-17T15:33:02.205+0000 7f2ae4eed880 -1 *** Caught signal (Aborted) **
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: in thread 7f2ae4eed880 thread_name:rbd-mirror
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f2ae5d80520]
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f2ae5dcf197]
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 4: rbd-mirror(+0x3726ab) [0x55807a87a6ab]
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::shut_down()+0x103) [0x55807a883b73]
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::~PoolReplayer()+0x10) [0x55807a883f10]
2024-05-17T15:33:02.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 7: rbd-mirror(+0x944a08) [0x55807ae4ca08]
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 8: rbd-mirror(+0x9449f0) [0x55807ae4c9f0]
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 9: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55807a87489a]
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 10: main()
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 11: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f2ae5d67d90]
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 12: __libc_start_main()
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: 13: _start()
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:33:02.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:
2024-05-17T15:33:02.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi171.stderr:daemon-helper: command crashed with signal 6
2024-05-17T15:33:07.327 INFO:tasks.daemonwatchdog.daemon_watchdog:thrasher.rbd_mirror.[cluster1] failed
2024-05-17T15:33:07.328 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons
2024-05-17T15:33:07.832 INFO:tasks.ceph.cluster1.osd.0:Sent signal 15
2024-05-17T15:33:07.833 INFO:tasks.ceph.cluster1.osd.1:Sent signal 15
2024-05-17T15:33:07.833 INFO:tasks.ceph.cluster1.osd.2:Sent signal 15
2024-05-17T15:33:07.833 INFO:tasks.ceph.cluster1.mon.a:Sent signal 15
2024-05-17T15:33:07.833 INFO:tasks.ceph.cluster1.mgr.x:Sent signal 15
2024-05-17T15:33:07.834 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:33:07.829+0000 7fc16ac0b640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-osd -f --cluster cluster1 -i 0  (PID: 24148) UID: 0
2024-05-17T15:33:07.834 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:33:07.829+0000 7fc16ac0b640 -1 osd.0 271 *** Got signal Terminated ***
2024-05-17T15:33:07.834 INFO:tasks.ceph.cluster1.osd.0.smithi072.stderr:2024-05-17T15:33:07.829+0000 7fc16ac0b640 -1 osd.0 271 *** Immediate shutdown (osd_fast_shutdown=true) ***
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f076a1b1640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-osd -f --cluster cluster1 -i 1  (PID: 24151) UID: 0
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f076a1b1640 -1 osd.1 271 *** Got signal Terminated ***
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.osd.1.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f076a1b1640 -1 osd.1 271 *** Immediate shutdown (osd_fast_shutdown=true) ***
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T15:33:07.829+0000 7fd088aeb640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-osd -f --cluster cluster1 -i 2  (PID: 24152) UID: 0
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T15:33:07.829+0000 7fd088aeb640 -1 osd.2 271 *** Got signal Terminated ***
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.osd.2.smithi072.stderr:2024-05-17T15:33:07.829+0000 7fd088aeb640 -1 osd.2 271 *** Immediate shutdown (osd_fast_shutdown=true) ***
2024-05-17T15:33:07.835 INFO:tasks.ceph.cluster1.mon.a.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f14a9964640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-mon -f --cluster cluster1 -i a  (PID: 23975) UID: 0
2024-05-17T15:33:07.836 INFO:tasks.ceph.cluster1.mon.a.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f14a9964640 -1 mon.a@0(leader) e1 *** Got Signal Terminated ***
2024-05-17T15:33:07.836 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f86128ad640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-mgr -f --cluster cluster1 -i x  (PID: 23978) UID: 0
2024-05-17T15:33:07.836 INFO:tasks.ceph.cluster1.mgr.x.smithi072.stderr:2024-05-17T15:33:07.829+0000 7f86128ad640 -1 mgr handle_mgr_signal  *** Got signal Terminated ***
2024-05-17T15:34:52.325 INFO:tasks.rbd_mirror_thrash:Failed to stop cluster2.client.mirror.0
2024-05-17T15:34:52.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 6
2024-05-17T15:34:52.327 ERROR:tasks.rbd_mirror_thrash:exception:
Traceback (most recent call last):
  File "/home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa/tasks/rbd_mirror_thrash.py", line 84, in _run
    self.do_thrash()
  File "/home/teuthworker/src/github.com_ceph_ceph-c_ee59d86aad04089481e55c1a15aa5871339935b4/qa/tasks/rbd_mirror_thrash.py", line 156, in do_thrash
    run.wait([daemon.proc], timeout=600)
  File "/home/teuthworker/src/git.ceph.com_teuthology_cad7edd8d899a7c59a48f7da3294fca4cb778f64/teuthology/orchestra/run.py", line 473, in wait
    check_time()
  File "/home/teuthworker/src/git.ceph.com_teuthology_cad7edd8d899a7c59a48f7da3294fca4cb778f64/teuthology/contextutil.py", line 134, in __call__
    raise MaxWhileTries(error_msg)
teuthology.exceptions.MaxWhileTries: reached maximum tries (101) after waiting for 600 seconds
2024-05-17T15:34:52.328 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:*** Caught signal (Aborted) **
2024-05-17T15:34:52.328 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: in thread 7f7544024880 thread_name:rbd-mirror
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7544eb7520]
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f7544f06197]
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 4: /lib/librados.so.2(+0x124b23) [0x7f75465cfb23]
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map<long, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> >, std::less<long>, std::allocator<std::pair<long const, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> > > > >*)+0x38) [0x561012ca0778]
2024-05-17T15:34:52.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x561012ca2508]
2024-05-17T15:34:52.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 7: (rbd::mirror::Mirror::run()+0xb5) [0x561012c78cc5]
2024-05-17T15:34:52.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 8: main()
2024-05-17T15:34:52.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f7544e9ed90]
2024-05-17T15:34:52.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 10: __libc_start_main()
2024-05-17T15:34:52.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 11: _start()
2024-05-17T15:34:52.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:2024-05-17T15:34:52.327+0000 7f7544024880 -1 *** Caught signal (Aborted) **
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: in thread 7f7544024880 thread_name:rbd-mirror
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7544eb7520]
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f7544f06197]
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 4: /lib/librados.so.2(+0x124b23) [0x7f75465cfb23]
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map<long, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> >, std::less<long>, std::allocator<std::pair<long const, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> > > > >*)+0x38) [0x561012ca0778]
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x561012ca2508]
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 7: (rbd::mirror::Mirror::run()+0xb5) [0x561012c78cc5]
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 8: main()
2024-05-17T15:34:52.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f7544e9ed90]
2024-05-17T15:34:52.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 10: __libc_start_main()
2024-05-17T15:34:52.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 11: _start()
2024-05-17T15:34:52.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:34:52.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:
2024-05-17T15:34:52.339 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: -1679> 2024-05-17T15:24:39.187+0000 7f7540fdf640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0  (PID: 48335) UID: 1000
2024-05-17T15:34:52.344 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:     0> 2024-05-17T15:34:52.327+0000 7f7544024880 -1 *** Caught signal (Aborted) **
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: in thread 7f7544024880 thread_name:rbd-mirror
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7544eb7520]
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f7544f06197]
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 4: /lib/librados.so.2(+0x124b23) [0x7f75465cfb23]
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map<long, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> >, std::less<long>, std::allocator<std::pair<long const, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> > > > >*)+0x38) [0x561012ca0778]
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x561012ca2508]
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 7: (rbd::mirror::Mirror::run()+0xb5) [0x561012c78cc5]
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 8: main()
2024-05-17T15:34:52.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f7544e9ed90]
2024-05-17T15:34:52.346 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 10: __libc_start_main()
2024-05-17T15:34:52.346 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 11: _start()
2024-05-17T15:34:52.346 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:34:52.346 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:
2024-05-17T15:34:52.348 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: -3383> 2024-05-17T15:24:39.187+0000 7f7540fdf640 -1 received  signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0  (PID: 48335) UID: 1000
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: -3382> 2024-05-17T15:34:52.327+0000 7f7544024880 -1 *** Caught signal (Aborted) **
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: in thread 7f7544024880 thread_name:rbd-mirror
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: ceph version 19.0.0-3735-gee59d86a (ee59d86aad04089481e55c1a15aa5871339935b4) squid (dev)
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7544eb7520]
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 2: /lib/x86_64-linux-gnu/libc.so.6(+0x91197) [0x7f7544f06197]
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 3: pthread_cond_wait()
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 4: /lib/librados.so.2(+0x124b23) [0x7f75465cfb23]
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 5: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map<long, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> >, std::less<long>, std::allocator<std::pair<long const, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> > > > >*)+0x38) [0x561012ca0778]
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 6: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x561012ca2508]
2024-05-17T15:34:52.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 7: (rbd::mirror::Mirror::run()+0xb5) [0x561012c78cc5]
2024-05-17T15:34:52.351 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 8: main()
2024-05-17T15:34:52.351 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 9: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f7544e9ed90]
2024-05-17T15:34:52.351 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 10: __libc_start_main()
2024-05-17T15:34:52.351 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: 11: _start()
2024-05-17T15:34:52.351 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2024-05-17T15:34:52.351 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:
2024-05-17T15:34:52.729 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi171.stderr:daemon-helper: command crashed with signal 6
2024-05-17T20:56:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:Terminated
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ cleanup 143
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local error_code=143
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ set +e
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ '[' 143 -ne 0 ']'
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ status
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ local cluster daemon image_pool image_ns image
2024-05-17T20:56:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for cluster in ${CLUSTER1} ${CLUSTER2}
2024-05-17T20:56:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 status
2024-05-17T20:56:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 status'
2024-05-17T20:56:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T20:56:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --cluster cluster1 -s
2024-05-17T21:01:56.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-17T21:01:56.523+0000 7fd6bc9ae640  0 monclient(hunting): authenticate timed out after 300
2024-05-17T21:01:56.520 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:[errno 110] RADOS timed out (error connecting to the cluster)
2024-05-17T21:01:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T21:01:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --cluster cluster1 service dump
2024-05-17T21:06:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-17T21:06:56.641+0000 7f1d07211640  0 monclient(hunting): authenticate timed out after 300
2024-05-17T21:06:56.643 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:[errno 110] RADOS timed out (error connecting to the cluster)
2024-05-17T21:06:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T21:06:56.645 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --cluster cluster1 service status
2024-05-17T21:11:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-17T21:11:56.767+0000 7f3c886c5640  0 monclient(hunting): authenticate timed out after 300
2024-05-17T21:11:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:[errno 110] RADOS timed out (error connecting to the cluster)
2024-05-17T21:11:56.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:11:56.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:11:56.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_pool in ${POOL} ${PARENT_POOL}
2024-05-17T21:11:56.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-17T21:11:56.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror  images
2024-05-17T21:11:56.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror  images'
2024-05-17T21:11:56.768 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' ls -l
2024-05-17T21:16:56.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T21:16:56.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-17T21:16:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:16:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:16:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror mirror pool info
2024-05-17T21:16:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror mirror pool info'
2024-05-17T21:16:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' mirror pool info
2024-05-17T21:21:56.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T21:21:56.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:21:56.850 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:21:56.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror mirror pool status
2024-05-17T21:21:56.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror mirror pool status'
2024-05-17T21:21:56.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T21:21:56.851 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' mirror pool status --verbose
2024-05-17T21:26:56.887 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T21:26:56.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:26:56.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:26:56.890 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster1 -p mirror --namespace '' ls
2024-05-17T21:31:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror  rbd_mirroring omap vals
2024-05-17T21:31:56.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror  rbd_mirroring omap vals'
2024-05-17T21:31:56.955 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror --namespace '' listomapvals rbd_mirroring
2024-05-17T21:36:56.960 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T21:36:56.961 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror  rbd_mirror_leader omap vals
2024-05-17T21:36:56.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror  rbd_mirror_leader omap vals'
2024-05-17T21:36:56.962 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror --namespace '' listomapvals rbd_mirror_leader
2024-05-17T21:41:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T21:41:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:41:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:41:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-17T21:41:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror ns1 images
2024-05-17T21:41:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror ns1 images'
2024-05-17T21:41:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns1 ls -l
2024-05-17T21:46:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T21:46:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-17T21:46:57.032 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:46:57.032 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirrorns1 mirror pool info
2024-05-17T21:46:57.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:46:57.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirrorns1 mirror pool info'
2024-05-17T21:46:57.033 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns1 mirror pool info
2024-05-17T21:51:57.071 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T21:51:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:51:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:51:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirrorns1 mirror pool status
2024-05-17T21:51:57.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirrorns1 mirror pool status'
2024-05-17T21:51:57.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T21:51:57.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns1 mirror pool status --verbose
2024-05-17T21:56:57.108 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T21:56:57.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T21:56:57.112 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T21:56:57.113 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster1 -p mirror --namespace ns1 ls
2024-05-17T22:01:57.152 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror ns1 rbd_mirroring omap vals
2024-05-17T22:01:57.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror ns1 rbd_mirroring omap vals'
2024-05-17T22:01:57.153 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror --namespace ns1 listomapvals rbd_mirroring
2024-05-17T22:06:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T22:06:57.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror ns1 rbd_mirror_leader omap vals
2024-05-17T22:06:57.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror ns1 rbd_mirror_leader omap vals'
2024-05-17T22:06:57.203 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror --namespace ns1 listomapvals rbd_mirror_leader
2024-05-17T22:11:57.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T22:11:57.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:11:57.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:11:57.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-17T22:11:57.228 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror ns2 images
2024-05-17T22:11:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror ns2 images'
2024-05-17T22:11:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns2 ls -l
2024-05-17T22:16:57.266 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T22:16:57.267 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-17T22:16:57.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:16:57.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirrorns2 mirror pool info
2024-05-17T22:16:57.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:16:57.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirrorns2 mirror pool info'
2024-05-17T22:16:57.268 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns2 mirror pool info
2024-05-17T22:21:57.307 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T22:21:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:21:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:21:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirrorns2 mirror pool status'
2024-05-17T22:21:57.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirrorns2 mirror pool status
2024-05-17T22:21:57.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T22:21:57.310 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns2 mirror pool status --verbose
2024-05-17T22:26:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T22:26:57.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:26:57.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:26:57.349 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster1 -p mirror --namespace ns2 ls
2024-05-17T22:31:57.393 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror ns2 rbd_mirroring omap vals
2024-05-17T22:31:57.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror ns2 rbd_mirroring omap vals'
2024-05-17T22:31:57.394 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror --namespace ns2 listomapvals rbd_mirroring
2024-05-17T22:36:57.425 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T22:36:57.446 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror ns2 rbd_mirror_leader omap vals
2024-05-17T22:36:57.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror ns2 rbd_mirror_leader omap vals'
2024-05-17T22:36:57.447 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror --namespace ns2 listomapvals rbd_mirror_leader
2024-05-17T22:41:57.459 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T22:41:57.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:41:57.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:41:57.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_pool in ${POOL} ${PARENT_POOL}
2024-05-17T22:41:57.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-17T22:41:57.460 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent  images
2024-05-17T22:41:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent  images'
2024-05-17T22:41:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace '' ls -l
2024-05-17T22:46:57.496 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T22:46:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-17T22:46:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:46:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:46:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent mirror pool info
2024-05-17T22:46:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent mirror pool info'
2024-05-17T22:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace '' mirror pool info
2024-05-17T22:51:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T22:51:57.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:51:57.537 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:51:57.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent mirror pool status
2024-05-17T22:51:57.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent mirror pool status'
2024-05-17T22:51:57.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T22:51:57.538 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace '' mirror pool status --verbose
2024-05-17T22:56:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T22:56:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T22:56:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T22:56:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster1 -p mirror_parent --namespace '' ls
2024-05-17T23:01:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent  rbd_mirroring omap vals
2024-05-17T23:01:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent  rbd_mirroring omap vals'
2024-05-17T23:01:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace '' listomapvals rbd_mirroring
2024-05-17T23:06:57.651 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T23:06:57.673 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent  rbd_mirror_leader omap vals
2024-05-17T23:06:57.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent  rbd_mirror_leader omap vals'
2024-05-17T23:06:57.674 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace '' listomapvals rbd_mirror_leader
2024-05-17T23:11:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T23:11:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:11:57.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:11:57.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-17T23:11:57.688 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent ns1 images
2024-05-17T23:11:57.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent ns1 images'
2024-05-17T23:11:57.689 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 ls -l
2024-05-17T23:16:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T23:16:57.748 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-17T23:16:57.749 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:16:57.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:16:57.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parentns1 mirror pool info
2024-05-17T23:16:57.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parentns1 mirror pool info'
2024-05-17T23:16:57.750 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 mirror pool info
2024-05-17T23:21:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T23:21:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:21:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:21:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parentns1 mirror pool status
2024-05-17T23:21:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parentns1 mirror pool status'
2024-05-17T23:21:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T23:21:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 mirror pool status --verbose
2024-05-17T23:26:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T23:26:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:26:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:26:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 ls
2024-05-17T23:31:57.846 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent ns1 rbd_mirroring omap vals
2024-05-17T23:31:57.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent ns1 rbd_mirroring omap vals'
2024-05-17T23:31:57.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns1 listomapvals rbd_mirroring
2024-05-17T23:36:57.878 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T23:36:57.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent ns1 rbd_mirror_leader omap vals
2024-05-17T23:36:57.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent ns1 rbd_mirror_leader omap vals'
2024-05-17T23:36:57.894 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns1 listomapvals rbd_mirror_leader
2024-05-17T23:41:57.913 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-17T23:41:57.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:41:57.914 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent ns2 images
2024-05-17T23:41:57.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:41:57.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-17T23:41:57.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent ns2 images'
2024-05-17T23:41:57.915 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 ls -l
2024-05-17T23:46:57.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T23:46:57.950 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-17T23:46:57.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:46:57.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parentns2 mirror pool info
2024-05-17T23:46:57.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:46:57.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parentns2 mirror pool info'
2024-05-17T23:46:57.952 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 mirror pool info
2024-05-17T23:51:57.981 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T23:51:57.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:51:57.984 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:51:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parentns2 mirror pool status
2024-05-17T23:51:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parentns2 mirror pool status'
2024-05-17T23:51:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-17T23:51:57.986 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 mirror pool status --verbose
2024-05-17T23:56:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-17T23:56:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-17T23:56:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-17T23:56:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 ls
2024-05-18T00:01:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent ns2 rbd_mirroring omap vals
2024-05-18T00:01:58.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent ns2 rbd_mirroring omap vals'
2024-05-18T00:01:58.057 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns2 listomapvals rbd_mirroring
2024-05-18T00:06:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T00:06:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster1 mirror_parent ns2 rbd_mirror_leader omap vals'
2024-05-18T00:06:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster1 mirror_parent ns2 rbd_mirror_leader omap vals
2024-05-18T00:06:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns2 listomapvals rbd_mirror_leader
2024-05-18T00:11:58.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T00:11:58.119 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T00:11:58.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T00:11:58.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for cluster in ${CLUSTER1} ${CLUSTER2}
2024-05-18T00:11:58.120 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 status
2024-05-18T00:11:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 status'
2024-05-18T00:11:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T00:11:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --cluster cluster2 -s
2024-05-18T00:16:58.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-18T00:16:58.252+0000 7f48c02c8640  0 monclient(hunting): authenticate timed out after 300
2024-05-18T00:16:58.232 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:[errno 110] RADOS timed out (error connecting to the cluster)
2024-05-18T00:16:58.234 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T00:16:58.234 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --cluster cluster2 service dump
2024-05-18T00:21:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-18T00:21:58.374+0000 7fee772b8640  0 monclient(hunting): authenticate timed out after 300
2024-05-18T00:21:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:[errno 110] RADOS timed out (error connecting to the cluster)
2024-05-18T00:21:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T00:21:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ ceph --cluster cluster2 service status
2024-05-18T00:26:58.482 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:2024-05-18T00:26:58.496+0000 7f21cb9f5640  0 monclient(hunting): authenticate timed out after 300
2024-05-18T00:26:58.483 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:[errno 110] RADOS timed out (error connecting to the cluster)
2024-05-18T00:26:58.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T00:26:58.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T00:26:58.485 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_pool in ${POOL} ${PARENT_POOL}
2024-05-18T00:26:58.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-18T00:26:58.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror  images'
2024-05-18T00:26:58.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror  images
2024-05-18T00:26:58.486 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace '' ls -l
2024-05-18T00:31:58.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T00:31:58.525 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-18T00:31:58.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T00:31:58.526 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T00:31:58.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror mirror pool info
2024-05-18T00:31:58.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror mirror pool info'
2024-05-18T00:31:58.527 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace '' mirror pool info
2024-05-18T00:36:58.564 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T00:36:58.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T00:36:58.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror mirror pool status
2024-05-18T00:36:58.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T00:36:58.566 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror mirror pool status'
2024-05-18T00:36:58.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T00:36:58.567 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace '' mirror pool status --verbose
2024-05-18T00:41:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T00:41:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T00:41:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T00:41:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster2 -p mirror --namespace '' ls
2024-05-18T00:46:58.647 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror  rbd_mirroring omap vals
2024-05-18T00:46:58.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror  rbd_mirroring omap vals'
2024-05-18T00:46:58.648 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror --namespace '' listomapvals rbd_mirroring
2024-05-18T00:51:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T00:51:58.705 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror  rbd_mirror_leader omap vals
2024-05-18T00:51:58.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror  rbd_mirror_leader omap vals'
2024-05-18T00:51:58.706 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror --namespace '' listomapvals rbd_mirror_leader
2024-05-18T00:56:58.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T00:56:58.718 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T00:56:58.719 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T00:56:58.719 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-18T00:56:58.719 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror ns1 images
2024-05-18T00:56:58.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror ns1 images'
2024-05-18T00:56:58.720 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns1 ls -l
2024-05-18T01:01:58.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T01:01:58.762 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-18T01:01:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:01:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirrorns1 mirror pool info
2024-05-18T01:01:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:01:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirrorns1 mirror pool info'
2024-05-18T01:01:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns1 mirror pool info
2024-05-18T01:06:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T01:06:58.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:06:58.820 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirrorns1 mirror pool status
2024-05-18T01:06:58.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:06:58.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirrorns1 mirror pool status'
2024-05-18T01:06:58.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T01:06:58.821 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns1 mirror pool status --verbose
2024-05-18T01:11:58.845 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T01:11:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:11:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:11:58.848 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster2 -p mirror --namespace ns1 ls
2024-05-18T01:16:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror ns1 rbd_mirroring omap vals
2024-05-18T01:16:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror ns1 rbd_mirroring omap vals'
2024-05-18T01:16:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror --namespace ns1 listomapvals rbd_mirroring
2024-05-18T01:21:58.920 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T01:21:58.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror ns1 rbd_mirror_leader omap vals
2024-05-18T01:21:58.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror ns1 rbd_mirror_leader omap vals'
2024-05-18T01:21:58.921 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror --namespace ns1 listomapvals rbd_mirror_leader
2024-05-18T01:26:58.956 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T01:26:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:26:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:26:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-18T01:26:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror ns2 images
2024-05-18T01:26:58.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror ns2 images'
2024-05-18T01:26:58.958 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns2 ls -l
2024-05-18T01:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T01:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-18T01:31:58.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:31:58.996 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirrorns2 mirror pool info
2024-05-18T01:31:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:31:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirrorns2 mirror pool info'
2024-05-18T01:31:58.997 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns2 mirror pool info
2024-05-18T01:36:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T01:36:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:36:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirrorns2 mirror pool status
2024-05-18T01:36:59.035 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:36:59.035 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirrorns2 mirror pool status'
2024-05-18T01:36:59.035 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T01:36:59.035 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns2 mirror pool status --verbose
2024-05-18T01:41:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T01:41:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:41:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:41:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster2 -p mirror --namespace ns2 ls
2024-05-18T01:46:59.117 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror ns2 rbd_mirroring omap vals
2024-05-18T01:46:59.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror ns2 rbd_mirroring omap vals'
2024-05-18T01:46:59.118 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror --namespace ns2 listomapvals rbd_mirroring
2024-05-18T01:51:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T01:51:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror ns2 rbd_mirror_leader omap vals'
2024-05-18T01:51:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror ns2 rbd_mirror_leader omap vals
2024-05-18T01:51:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror --namespace ns2 listomapvals rbd_mirror_leader
2024-05-18T01:56:59.187 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T01:56:59.188 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T01:56:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T01:56:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_pool in ${POOL} ${PARENT_POOL}
2024-05-18T01:56:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-18T01:56:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parent  images
2024-05-18T01:56:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parent  images'
2024-05-18T01:56:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace '' ls -l
2024-05-18T02:01:59.224 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T02:01:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-18T02:01:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:01:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parent mirror pool info
2024-05-18T02:01:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:01:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parent mirror pool info'
2024-05-18T02:01:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace '' mirror pool info
2024-05-18T02:06:59.258 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T02:06:59.260 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:06:59.260 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:06:59.260 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parent mirror pool status
2024-05-18T02:06:59.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parent mirror pool status'
2024-05-18T02:06:59.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T02:06:59.261 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace '' mirror pool status --verbose
2024-05-18T02:11:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T02:11:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:11:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:11:59.296 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster2 -p mirror_parent --namespace '' ls
2024-05-18T02:16:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parent  rbd_mirroring omap vals'
2024-05-18T02:16:59.334 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parent  rbd_mirroring omap vals
2024-05-18T02:16:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror_parent --namespace '' listomapvals rbd_mirroring
2024-05-18T02:21:59.363 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T02:21:59.364 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parent  rbd_mirror_leader omap vals
2024-05-18T02:21:59.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parent  rbd_mirror_leader omap vals'
2024-05-18T02:21:59.365 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rados --cluster cluster2 -p mirror_parent --namespace '' listomapvals rbd_mirror_leader
2024-05-18T02:26:59.399 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:failed to fetch mon config (--no-mon-config to skip)
2024-05-18T02:26:59.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:26:59.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:26:59.400 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ for image_ns in "" "${NS1}" "${NS2}"
2024-05-18T02:26:59.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parent ns1 images
2024-05-18T02:26:59.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parent ns1 images'
2024-05-18T02:26:59.401 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace ns1 ls -l
2024-05-18T02:31:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T02:31:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: listing images failed: (110) Connection timed out
2024-05-18T02:31:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:31:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parentns1 mirror pool info
2024-05-18T02:31:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:31:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parentns1 mirror pool info'
2024-05-18T02:31:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace ns1 mirror pool info
2024-05-18T02:36:59.470 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T02:36:59.473 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:36:59.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:36:59.474 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:cluster2 mirror_parentns1 mirror pool status
2024-05-18T02:36:59.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo 'cluster2 mirror_parentns1 mirror pool status'
2024-05-18T02:36:59.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ CEPH_ARGS=
2024-05-18T02:36:59.477 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace ns1 mirror pool status --verbose
2024-05-18T02:41:59.522 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:rbd: couldn't connect to the cluster!
2024-05-18T02:41:59.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stdout:
2024-05-18T02:41:59.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:++ echo
2024-05-18T02:41:59.524 INFO:tasks.workunit.cluster1.client.mirror.smithi171.stderr:+++ rbd --cluster cluster2 -p mirror_parent --namespace ns1 ls
2024-05-18T02:43:23.596 DEBUG:teuthology.exit:Got signal 15; running 1 handler...
2024-05-18T02:43:23.597 DEBUG:teuthology.task.console_log:Killing console logger for smithi072
2024-05-18T02:43:23.597 DEBUG:teuthology.task.console_log:Killing console logger for smithi171
2024-05-18T02:43:23.598 DEBUG:teuthology.exit:Finished running handlers